do not reconfigure apps when infra version has not changed

fix merging blooper
assume success if dns server to be down
2016-07-25 18:57:54 -07:00 · 2016-07-25 16:18:09 -07:00 · 2016-07-25 15:31:26 -07:00 · 2016-07-25 14:38:19 -07:00 · 2016-07-25 14:19:20 -07:00 · 2016-07-25 12:54:27 -07:00
234 changed files with 26144 additions and 7285 deletions
@@ -1,6 +1,7 @@
-# Skip files when using git archive
+# following files are skipped when exporting using git archive
+/release export-ignore
+/admin export-ignore
+test export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
-/scripts export-ignore
-test export-ignore

@@ -3,7 +3,9 @@ coverage/
 docs/
 webadmin/dist/
 setup/splash/website/
+installer/src/certs/server.key

 # vim swap files
 *.swp

+
@@ -1,7 +1,8 @@
 {
-	"node": true,
-	"browser": true,
-	"unused": true,
-	"globalstrict": true,
-	"predef": [ "angular", "$" ]
+    "node": true,
+    "browser": true,
+    "unused": true,
+    "globalstrict": true,
+    "predef": [ "angular", "$" ],
+    "esnext": true
 }
@@ -0,0 +1,585 @@
+[0.0.1]
+- Hot Chocolate
+
+[0.0.2]
+- Hotfix appstore ui in webadim
+
+[0.0.3]
+- Tall Pike
+
+[0.0.4]
+- This will be 0.0.4 changes
+
+[0.0.5]
+- App install/configure route fixes
+
+[0.0.6]
+- Not sure what happenned here
+
+[0.0.7]
+- resetToken is now sent as part of create user
+- Same as 0.0.7 which got released by mistake
+
+[0.0.8]
+- Manifest changes
+
+[0.0.9]
+- Fix app restore
+- Fix backup issues
+
+[0.0.10]
+- Unknown orchestra
+
+[0.0.11]
+- Add ldap addon
+
+[0.0.12]
+- Support OAuth2 state
+
+[0.0.13]
+- Use docker image from cloudron repository
+
+[0.0.14]
+- Improve setup flow
+
+[0.0.15]
+- Improved Appstore view
+
+[0.0.16]
+- Improved Backup approach
+
+[0.0.17]
+- Upgrade testing
+- App auto updates
+- Usage graphs
+
+[0.0.18]
+- Rework backups and updates
+
+[0.0.19]
+- Graphite fixes
+- Avatar and Cloudron name support
+
+[0.0.20]
+- Apptask fixes
+- Chrome related fixes
+
+[0.0.21]
+- Increase nginx hostname size to 64
+
+[0.0.22]
+- Testing the e2e tests
+
+[0.0.23]
+- Better error status page
+- Fix updater and backup progress reporting
+- New avatar set
+- Improved setup wizard
+
+[0.0.24]
+- Hotfix the ldap support
+
+[0.0.25]
+- Add support page
+- Really fix ldap issues
+
+[0.0.26]
+- Add configurePath support
+
+[0.0.27]
+- Improved log collector
+
+[0.0.28]
+- Improve app feedback
+- Restyle login page
+
+[0.0.29]
+- Update to ubuntu 15.04
+
+[0.0.30]
+- Move to docker 1.7
+
+[0.0.31]
+- WARNING: This update restarts your containers
+- System processes are prioritized over apps
+- Add ldap group support
+
+[0.0.32]
+- MySQL addon update
+
+[0.0.33]
+- Fix graphs
+- Fix MySQL 5.6 memory usage
+
+[0.0.34]
+- Correctly mark apps pending for approval
+
+[0.0.35]
+- Fix ldap admin group username
+
+[0.0.36]
+- Fix restore without backup
+- Optimize image deletion during updates
+- Add memory accounting
+- Restrict access to metadata from containers
+
+[0.0.37]
+- Prepare for Selfhosting 1. part
+- Use userData instead of provisioning calls
+
+[0.0.38]
+- Account for Ext4 reserved block when partitioning disk
+
+[0.0.39]
+- Move subdomain management to the cloudron
+
+[0.0.40]
+- Add journal limit
+- Fix reprovisioning on reboot
+- Fix subdomain management during startup
+
+[0.0.41]
+- Finally bring things to a sane state
+
+[0.0.42]
+- Parallel apptask
+
+[0.0.43]
+- Move to systemd
+
+[0.0.44]
+- Fix apptask concurrency bug
+
+[0.0.45]
+- Retry subdomain registration
+
+[0.0.46]
+- Fix app update email notification
+
+[0.0.47]
+- Ensure box code quits within 5 seconds
+
+[0.0.48]
+- Styling fixes
+- Improved session handling
+
+[0.0.49]
+- Fix app autoupdate logic
+
+[0.0.50]
+- Use domainmanagement via CaaS
+
+[0.0.51]
+- Fix memory management
+
+[0.0.52]
+- Restrict addons memory
+- Get nofication about container OOMs
+
+[0.0.53]
+- Restrict addons memory
+- Get notification about container OOMs
+- Add retry to subdomain logic
+
+[0.0.54]
+- OAuth Proxy now uses internal port forwarding
+
+[0.0.55]
+- Setup cloudron timezone based on droplet region
+
+[0.0.56]
+- Use correct timezone in updater
+
+[0.0.57]
+- Fix systemd logging issues
+
+[0.0.58]
+- Ensure backups of failed apps are retained across archival cycles
+
+[0.0.59]
+- Installer API fixes
+
+[0.0.60]
+- Do full box backup on updates
+
+[0.0.61]
+- Track update notifications to inform admin only once
+
+[0.0.62]
+- Export bind dn and password from LDAP addon
+
+[0.0.63]
+- Fix creation of TXT records
+
+[0.0.64]
+- Stop apps in a retired cloudron
+- Retry downloading application on failure
+
+[0.0.65]
+- Do not send crash mails for apps in development
+
+[0.0.66]
+- Readonly application and addon containers
+
+[0.0.67]
+- Fix email notifications
+- Fix bug when restoring from certain backups
+
+[0.0.68]
+- Update graphite image
+- Add simpleauth addon support
+
+[0.0.69]
+- Support newer manifest format
+- Fix app listing rendering in chrome
+- Fix redis backup across upgrades
+
+[0.0.70]
+- Retry app download on error
+
+[0.0.71]
+- Fix oauth and simple auth login
+
+[0.0.72]
+- Cleanup application volumes periodically
+- New application logging design
+
+[0.0.73]
+- Update SSL certificate
+
+[0.0.74]
+- Support singleUser apps
+
+[0.0.75]
+- scheduler addon
+
+[0.0.76]
+- DNS Sync fixes
+- Show warning to user when memory limit reached
+
+[0.0.77]
+- Do not set hostname in app containers
+
+[0.0.78]
+- Support custom domains
+
+[0.0.79]
+- Move SSH Port
+
+[0.0.80]
+- Use journalctl for container logs
+
+[0.1.0]
+- Wait for configuration changes before starting Cloudron
+
+[0.1.1]
+- Ensure dns config for all cloudrons
+
+[0.1.2]
+- Make email work again
+- Add DKIM keys for custom domains
+
+[0.1.3]
+- Storage backend
+
+[0.1.4]
+- CaaS Backup configuration fix
+
+[0.1.5]
+- Use correct tokens for DNS backend
+
+[0.1.6]
+- Add hook to determine the api server of the box
+- Fix crash notification
+
+[0.2.0]
+- New cloudron exec implementation
+
+[0.2.1]
+- Update to node 4.1.1
+- Fix certification installation with custom domains
+
+[0.2.2]
+- Better debug output
+- Retry more times if docker registry goes down
+
+[0.3.0]
+- Update SSH keys
+- Allow bigger manifest files
+
+[0.4.0]
+- Update to docker 1.9.0
+
+[0.4.1]
+- Fix scheduler crash
+- Crucial OAuth fixes
+
+[0.4.2]
+- Fix crash when reporting backup error
+- Allow larger manifests
+
+[0.4.3]
+- Fix cloudron exec
+
+[0.4.4]
+- Initial Lets Encrypt integration
+
+[0.4.5]
+- Fixup nginx configuration to allow dynamic certificates
+
+[0.4.6]
+- LetsEncrypt integration for custom domains
+- Rate limit crash emails
+
+[0.5.0]
+- Enable staging Lets Encrypt Integration
+
+[0.5.1]
+- Display error dialog for app installation errors
+- Enable prod Lets Encrypt Integration
+- Handle apptask crashes correctly
+
+[0.5.2]
+- Fix apphealthtask crash
+- Use cgroup fs driver instead of systemd cgroup driver in docker
+
+[0.5.3]
+- Changes for e2e testing
+
+[0.5.4]
+- Fix bug in LE server selection
+
+[0.5.5]
+- Scheduler redesign
+- Fix journalctl logging
+
+[0.5.6]
+- Prepare for selfhosting option
+
+[0.5.7]
+- Move app images off the btrfs subvolume
+
+[0.6.0]
+- Consolidate code repositories
+
+[0.6.1]
+- Use no-reply as email from address for apps in naked domains
+- Update Lets Encrypt account with owner email when available
+- Fix email templates to indicate auto update
+- Add notification UI
+
+[0.6.2]
+- Fix `cloudron exec` container to have same namespaces as app
+- Add developmentMode to manifest
+
+[0.6.3]
+- Make sending invite for new users optional
+
+[0.6.4]
+- Add support for display names
+- Send invite links to admins for user setup
+- Enforce stronger passwords
+
+[0.6.5]
+- Finalize stronger password requirement
+
+[0.7.0]
+- Upgrade to 15.10
+- Do not remove docker images when in use by another container
+- Fix sporadic error when reconfiguring apps
+- Handle journald crashes gracefully
+
+[0.7.1]
+- Allow admins to edit users
+- Fix graphs
+- Support more LDAP cases
+- Allow appstore deep linking
+
+[0.7.2]
+- Fix 5xx errors when password does not meet requirements
+- Improved box update management using prereleases
+- Less aggressive disk space checks
+
+[0.8.0]
+- MySQL addon : multiple database support
+
+[0.8.1]
+- Set Host HTTP header when querying healthCheckPath
+- Show application Changelog in app update emails
+
+[0.9.0]
+- Fix bug in multdb mysql addon backup
+- Add initial user group support
+- Improved app memory limit handling
+
+[0.9.1]
+- Introduce per app group access control
+
+[0.9.2]
+- Fix bug where reconfiguring apps would trigger memory limit warning
+- Allow more apps to be installed in bigger sized cloudrons
+- Allow user to override memory limit warning and install anyway
+
+[0.9.3]
+- Admin flag is handled outside of groups
+- User interface fixes for groups
+- Allow to set access restrictions on app installation
+
+[0.10.0]
+- Upgrade to docker 1.10.2
+- Fix MySQL addon to handle heavier loads
+- Allow listing and download of backups (using the CLI tool)
+- Ubuntu security updates till 8th March 2016 (http://www.ubuntu.com/usn)
+
+[0.10.1]
+- Fix Let's Encrypt certificate renewal
+
+[0.10.2]
+- Apps can now bind with username or email with LDAP
+- Disallow updating an app with mismatching manifest id
+- Use admin domain instead of naked domain in the SPF record
+- Download Lets Encrypt intermediate cert
+
+[0.10.3]
+- Store the backup config for each backup. This will allow using multiple buckets/providers for backups simultaneously.
+- Fix SPF record check
+
+[0.10.4]
+- Fix restore for droplets in EU region
+
+[0.11.0]
+- Store backups in the same region as the Cloudron
+- Fix PCRE security issue (http://www.ubuntu.com/usn/usn-2943-1/)
+
+[0.11.1]
+- Improve the backup logic
+
+[0.11.2]
+- Allow users to choose a username on first sign up
+- Fix app graphs
+
+[0.12.0]
+- Fix upload of large backups
+- Postgres addon whitelists pg_trgm and hstore extensions
+- Suppress boring update emails from patch releases
+- Setup bounce alerts for emails
+- Query admin's name in activation wizard
+- Admin emails are now delivered as no-reply
+- Fix crash when user attempts to set a duplicate email
+- Improved mongodb crash recovery
+
+[0.12.1]
+- Fix crash when backing up apps
+
+[0.12.2]
+- Improved error handling for addons
+
+[0.12.3]
+- LDAP: Do not set sn attribute when user has no surname
+
+[0.12.4]
+- Install app only after platform is ready
+
+[0.12.5]
+- Get alerts for app task failures
+- Fix update issue when one or more apps are in failed state
+
+[0.12.6]
+- Allow setting an alternate external domain for apps
+
+[0.12.7]
+- Fix changing password
+
+[0.13.0]
+- Upgrade to ubuntu 16.04
+- Add event log
+
+[0.13.1]
+- Make activity log viewable to admins
+- Fix geoip lookup
+
+[0.13.2]
+- Fix crash in app auto updater
+- Fix crash with empty timezone
+
+[0.13.3]
+- Enable auth in email addon
+- Add search for activity log
+- Add tutorial for first time users
+
+[0.13.4]
+- Fix mail addon restart issue
+
+[0.14.0]
+- You have mail :-)
+
+[0.14.1]
+- 2-character usernames are now allowed
+- Make cloudron CLI push/pull more robust
+
+[0.14.2]
+- Update mail addon
+
+[0.15.0]
+- [REST API](https://cloudron.io/references/api.html) is now in public beta
+- Enable Developer mode by default for new Cloudrons
+- Reverse proxy fixes for apps exposing a WebDav server
+- Allow admins to optionally set the username and displayName on user creation
+- Fix app autoupdate logic to detect if one or more in-use port bindings was removed
+
+[0.15.1]
+- Fix mail connectivity from IPv6 clients
+- Add API token management UI
+- Improved UI to enter email aliases
+
+[0.15.2]
+- Allow restoring apps from any previous backup
+
+[0.15.3]
+- Show installation progress in a tooltip
+
+[0.16.0]
+- Allow apps to be configured in configuring state
+- Improved platform architecture that allows incremental infrastructure updates
+- Implement app clone
+
+[0.16.1]
+- Fix UI layout issue in tokens page
+- Resume app tasks only when configured and platform ready
+- Allow errored apps to be reconfigured
+
+[0.16.2]
+- Fix assert when backing up apps in errored state
+- Fix bug where multiple redis installations caused an error
+
+[0.16.3]
+- Timeout in 10mins if app restore fails because of external domain CNAME setup
+
+[0.16.4]
+- Setup email aliases to only alias names for the Cloudron domain
+
+[0.16.5]
+- Allow sending email with alias as the From
+
+[0.16.6]
+- Add plan migration interface
+- Initial EC2 support
+
+[0.17.0]
+- Public beta release of Cloudron Mail Server
+- Add new DNS & Certs UI that enables easy migration to a custom domain
+- Allow sending and receiving email from alias subaddresses
+- Fix installation issue with some apps on the naked domain
+
+[0.17.1]
+- Preliminary user impersonation support
+- Fix crash in mail container when generating bounces
+
+[0.17.2]
+- Add config option to embed apps in other sites
+
+[0.17.3]
+- Incremental infrastructure update logic
+- Keep eventlogs only for a week
+- Throttle OOM mails
+
@@ -1,10 +1,17 @@
-The Box
-=======
+Cloudron a Smart Server
+=======================

-Development setup
-----------------
-* sudo useradd -m yellowtent
-** Add admin-localhost as 127.0.0.1 in /etc/hosts
-** All apps will be installed as hypened-subdomains of localhost. You should add
-   hyphened-subdomains of your apps into /etc/hosts

+
+Selfhost Instructions
+---------------------
+
+The smart server currently relies on an AWS account with access to Route53 and S3 and is tested on DigitalOcean and EC2.
+
+First create a virtual private server with Ubuntu 15.04 and run the following commands in an ssh session to initialize the base image:
+
+```
+curl https://s3.amazonaws.com/prod-cloudron-releases/installer.sh -o installer.sh
+chmod +x installer.sh
+./installer.sh <domain> <aws access key> <aws acccess secret> <backup bucket> <provider> <release sha1>
+```
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+installer_revision=$(git rev-parse HEAD)
+box_name=""
+server_id=""
+server_ip=""
+destroy_server="yes"
+deploy_env="dev"
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --env) deploy_env="$2"; shift 2;;
+    --revision) installer_revision="$2"; shift 2;;
+    --name) box_name="$2"; destroy_server="no"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+echo "Creating digitalocean image"
+if [[ "${deploy_env}" == "staging" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
+elif [[ "${deploy_env}" == "dev" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
+elif [[ "${deploy_env}" == "prod" ]]; then
+    assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
+    export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
+else
+    echo "No such env ${deploy_env}."
+    exit 1
+fi
+
+vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}"
+readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${installer_revision}")
+
+if [[ -z "${box_name}" ]]; then
+    # if you change this, change the regexp is appstore/janitor.js
+    box_name="box-${deploy_env}-${pretty_revision}-${now}" # remove slashes
+
+    # create a new server if no name given
+    if ! caas_ssh_key_id=$($vps get_ssh_key_id "caas"); then
+        echo "Could not query caas ssh key"
+        exit 1
+    fi
+    echo "Detected caas ssh key id: ${caas_ssh_key_id}"
+
+    echo "Creating Server with name [${box_name}]"
+    if ! server_id=$($vps create ${caas_ssh_key_id} ${box_name}); then
+        echo "Failed to create server"
+        exit 1
+    fi
+    echo "Created server with id: ${server_id}"
+
+    # If we run scripts overenthusiastically without the wait, setup script randomly fails
+    echo -n "Waiting 120 seconds for server creation"
+    for i in $(seq 1 24); do
+        echo -n "."
+        sleep 5
+    done
+    echo ""
+else
+    if ! server_id=$($vps get_id "${box_name}"); then
+        echo "Could not determine id from name"
+        exit 1
+    fi
+    echo "Reusing server with id: ${server_id}"
+
+    $vps power_on "${server_id}"
+fi
+
+# Query until we get an IP
+while true; do
+    echo "Trying to get the server IP"
+    if server_ip=$($vps get_ip "${server_id}"); then
+        echo "Server IP : [${server_ip}]"
+        break
+    fi
+    echo "Timedout, trying again in 10 seconds"
+    sleep 10
+done
+
+while true; do
+    echo "Trying to copy init script to server"
+    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" root@${server_ip}:.; then
+        break
+    fi
+    echo "Timedout, trying again in 30 seconds"
+    sleep 30
+done
+
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" root@${server_ip}:.
+
+echo "Copying box source"
+cd "${SOURCE_DIR}"
+git archive --format=tar HEAD | $ssh22 "root@${server_ip}" "cat - > /tmp/box.tar.gz"
+
+echo "Executing init script"
+if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+    echo "Init script failed"
+    exit 1
+fi
+
+echo "Shutting down server with id : ${server_id}"
+$ssh202 "root@${server_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail
+
+# wait 10 secs for actual shutdown
+echo "Waiting for 10 seconds for server to shutdown"
+sleep 30
+
+echo "Powering off server"
+if ! $vps power_off "${server_id}"; then
+    echo "Could not power off server"
+    exit 1
+fi
+
+snapshot_name="box-${deploy_env}-${pretty_revision}-${now}"
+echo "Snapshotting as ${snapshot_name}"
+if ! image_id=$($vps snapshot "${server_id}" "${snapshot_name}"); then
+    echo "Could not snapshot and get image id"
+    exit 1
+fi
+
+if [[ "${destroy_server}" == "yes" ]]; then
+    echo "Destroying server"
+    if ! $vps destroy "${server_id}"; then
+        echo "Could not destroy server"
+        exit 1
+    fi
+else
+    echo "Skipping server destroy"
+fi
+
+echo "Transferring image ${image_id} to other regions"
+$vps transfer_image_to_all_regions "${image_id}"
+
+echo "Done."
@@ -0,0 +1,185 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+installer_revision=$(git rev-parse HEAD)
+instance_id=""
+server_ip=""
+destroy_server="yes"
+
+ami_id="ami-f9e30f96"
+region="eu-central-1"
+aws_credentials="baseimage"
+security_group="sg-b9a473d1"
+instance_type="t2.small"
+subnet_id="subnet-801402e9"
+key_pair_name="id_rsa_yellowtent"
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revisio0n:,no-destroy" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --revision) installer_revision="$2"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
+readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+function debug() {
+    echo "$@" >&2
+}
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${installer_revision}")
+
+echo "Creating EC2 instance"
+instance_id=$(aws ec2 run-instances --image-id ${ami_id} --region ${region} --profile ${aws_credentials} --security-group-ids ${security_group} --instance-type ${instance_type} --key-name ${key_pair_name} --subnet-id ${subnet_id} --associate-public-ip-address | $JSON Instances[0].InstanceId)
+echo "Got InstanceId: ${instance_id}"
+
+# name the instance
+aws ec2 create-tags --profile ${aws_credentials} --resources ${instance_id} --tags "Key=Name,Value=baseimage-${pretty_revision}"
+
+echo "Waiting for instance to be running..."
+while true; do
+    event_status=`aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].State.Name`
+    if [[ "${event_status}" == "running" ]]; then
+        break
+    fi
+    debug -n "."
+    sleep 10
+done
+
+server_ip=$(aws ec2 describe-instances --instance-id ${instance_id} --region ${region} --profile ${aws_credentials} | $JSON Reservations[0].Instances[0].PublicIpAddress)
+echo "Server IP is: ${server_ip}"
+
+while true; do
+    echo "Trying to copy init script to server"
+    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" ubuntu@${server_ip}:.; then
+        break
+    fi
+    echo "Timedout, trying again in 30 seconds"
+    sleep 30
+done
+
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" ubuntu@${server_ip}:.
+
+echo "Copying box source"
+cd "${SOURCE_DIR}"
+git archive --format=tar HEAD | $ssh22 "ubuntu@${server_ip}" "cat - > /tmp/box.tar.gz"
+
+echo "Enabling root ssh access"
+if ! $ssh22 "ubuntu@${server_ip}" "sudo sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys"; then
+    echo "Unable to enable root access"
+    echo "Make sure to cleanup the ec2 instance ${instance_id}"
+    exit 1
+fi
+
+echo "Executing init script"
+if ! $ssh22 "root@${server_ip}" "/bin/bash /home/ubuntu/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+    echo "Init script failed"
+    echo "Make sure to cleanup the ec2 instance ${instance_id}"
+    exit 1
+fi
+
+snapshot_name="cloudron-${pretty_revision}-${now}"
+
+echo "Creating ami image ${snapshot_name}"
+image_id=$(aws ec2 create-image --region ${region} --profile ${aws_credentials} --instance-id ${instance_id} --name ${snapshot_name} | $JSON ImageId)
+
+echo "Image creation started for image id: ${image_id}"
+
+echo "Waiting for image creation to finish..."
+while true; do
+    event_status=`aws ec2 describe-images --region ${region} --profile ${aws_credentials} --image-id ${image_id} | $JSON Images[0].State`
+    if [[ "${event_status}" == "available" ]]; then
+        break
+    fi
+    debug -n "."
+    sleep 10
+done
+
+echo "Terminating instance"
+aws ec2 terminate-instances --region ${region} --profile ${aws_credentials} --instance-ids ${instance_id}
+
+echo "Make image public"
+aws ec2 modify-image-attribute --region ${region} --profile ${aws_credentials} --image-id ${image_id} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+
+
+# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
+# Images are currently created in eu-central-1
+echo "Coping image to other regions"
+ec2_regions=( "us-east-1" "us-west-1" "us-west-2" "ap-south-1" "ap-northeast-2" "ap-southeast-1" "ap-southeast-2" "ap-northeast-1" "eu-west-1" "sa-east-1" )
+ec2_amis=( )
+
+for r in ${ec2_regions[@]}; do
+    echo "=> ${r}"
+    ami_id=$(aws ec2 copy-image --region ${r} --profile ${aws_credentials} --source-image-id ${image_id} --source-region ${region} --name ${snapshot_name} | $JSON ImageId)
+
+    # append in the same order as the regions
+    ec2_amis+=( ${ami_id} )
+done
+
+# wait for all images to be available
+echo "Waiting for images to be ready (first will take the longest)..."
+region_string="${region}=${image_id}"
+i=0
+while [ $i -lt ${#ec2_regions[*]} ]; do
+    echo "=> ${ec2_regions[$i]} ${ec2_amis[$i]}"
+    while true; do
+        event_status=`aws ec2 describe-images --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} | $JSON Images[0].State`
+        if [[ "${event_status}" == "available" ]]; then
+            echo "done"
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+
+    # now make it public
+    aws ec2 modify-image-attribute --region ${ec2_regions[$i]} --profile ${aws_credentials} --image-id ${ec2_amis[$i]} --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+
+    # append to output string for release tool
+    region_string+=",${ec2_regions[$i]}=${ec2_amis[$i]}"
+
+    # inc the iteration counter
+    i=$(( $i + 1));
+done
+
+echo ""
+echo "--------------------------------------------------"
+echo "New image id is: ${image_id}"
+echo "Image region string for release:"
+echo "${region_string}"
+echo "--------------------------------------------------"
+echo ""
@@ -0,0 +1,261 @@
+#!/bin/bash
+
+if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then
+    echo "Script requires DIGITAL_OCEAN_TOKEN env to be set"
+    exit 1
+fi
+
+if [[ -z "${JSON}" ]]; then
+    echo "Script requires JSON env to be set to path of JSON binary"
+    exit 1
+fi
+
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
+
+function debug() {
+    echo "$@" >&2
+}
+
+function get_ssh_key_id() {
+    id=$($CURL "https://api.digitalocean.com/v2/account/keys" \
+        | $JSON ssh_keys \
+        | $JSON -c "this.name === \"$1\"" \
+        | $JSON 0.id)
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function create_droplet() {
+    local ssh_key_id="$1"
+    local box_name="$2"
+
+    local image_region="sfo1"
+    local ubuntu_image_slug="ubuntu-16-04-x64"
+    local box_size="512mb"
+
+    local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
+
+    id=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id)
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function get_droplet_ip() {
+    local droplet_id="$1"
+    ip=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address")
+    [[ -z "$ip" ]] && exit 1
+    echo "$ip"
+}
+
+function get_droplet_id() {
+    local droplet_name="$1"
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function power_off_droplet() {
+    local droplet_id="$1"
+    local data='{"type":"power_off"}'
+    local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions")
+    local event_id=`echo "${response}" | $JSON action.id`
+
+    if [[ -z "${event_id}" ]]; then
+        debug "Got no event id, assuming already powered off."
+        debug "Response: ${response}"
+        return
+    fi
+
+    debug "Powered off droplet. Event id: ${event_id}"
+    debug -n "Waiting for droplet to power off"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function power_on_droplet() {
+    local droplet_id="$1"
+    local data='{"type":"power_on"}'
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
+
+    debug "Powered on droplet. Event id: ${event_id}"
+
+    if [[ -z "${event_id}" ]]; then
+        debug "Got no event id, assuming already powered on"
+        return
+    fi
+
+    debug -n "Waiting for droplet to power on"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function get_image_id() {
+    local snapshot_name="$1"
+    local image_id=""
+
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
+    fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+        return 1
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
+}
+
+function snapshot_droplet() {
+    local droplet_id="$1"
+    local snapshot_name="$2"
+    local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}"
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
+
+    debug "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}"
+    debug -n "Waiting for snapshot to complete"
+
+    while true; do
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug "! done"
+
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
+}
+
+function destroy_droplet() {
+    local droplet_id="$1"
+    # TODO: check for 204 status
+    $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}"
+    debug "Droplet destroyed"
+    debug ""
+}
+
+function transfer_image() {
+    local image_id="$1"
+    local region_slug="$2"
+    local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}"
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id`
+    echo "${event_id}"
+}
+
+function wait_for_image_event() {
+    local image_id="$1"
+    local event_id="$2"
+
+    debug -n "Waiting for ${event_id}"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function transfer_image_to_all_regions() {
+    local image_id="$1"
+
+    xfer_events=()
+    image_regions=(ams2) ## sfo1 is where the image is created
+    for image_region in ${image_regions[@]}; do
+        xfer_event=$(transfer_image ${image_id} ${image_region})
+        echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"
+        xfer_events+=("${xfer_event}")
+        sleep 1
+    done
+
+    echo "Image transfer initiated, but they will take some time to get transferred."
+
+    for xfer_event in ${xfer_events[@]}; do
+        $vps wait_for_image_event "${image_id}" "${xfer_event}"
+    done
+}
+
+if [[ $# -lt 1 ]]; then
+    debug "<command> <params...>"
+    exit 1
+fi
+
+case $1 in
+get_ssh_key_id)
+    get_ssh_key_id "${@:2}"
+    ;;
+
+create)
+    create_droplet "${@:2}"
+    ;;
+
+get_id)
+    get_droplet_id "${@:2}"
+    ;;
+
+get_ip)
+    get_droplet_ip "${@:2}"
+    ;;
+
+power_on)
+    power_on_droplet "${@:2}"
+    ;;
+
+power_off)
+    power_off_droplet "${@:2}"
+    ;;
+
+snapshot)
+    snapshot_droplet "${@:2}"
+    ;;
+
+destroy)
+    destroy_droplet "${@:2}"
+    ;;
+
+transfer_image_to_all_regions)
+    transfer_image_to_all_regions "${@:2}"
+    ;;
+
+*)
+    echo "Unknown command $1"
+    exit 1
+esac
@@ -0,0 +1,286 @@
+#!/bin/bash
+
+set -euv -o pipefail
+
+readonly USER=yellowtent
+readonly USER_HOME="/home/${USER}"
+readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
+readonly INSTALLER_REVISION="$1"
+readonly USER_DATA_FILE="/root/user_data.img"
+readonly USER_DATA_DIR="/home/yellowtent/data"
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+function die {
+    echo $1
+    exit 1
+}
+
+[[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"
+
+echo "==== Create User ${USER} ===="
+if ! id "${USER}"; then
+    useradd "${USER}" -m
+fi
+
+echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ==="
+
+echo "=== Prepare installer source ==="
+rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}"
+rm -rf /tmp/box && mkdir -p /tmp/box
+tar xvf /tmp/box.tar.gz -C /tmp/box && rm /tmp/box.tar.gz
+cp -rf /tmp/box/installer/* "${INSTALLER_SOURCE_DIR}"
+echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION"
+
+export DEBIAN_FRONTEND=noninteractive
+
+echo "=== Upgrade ==="
+apt-get update
+apt-get dist-upgrade -y
+apt-get install -y curl
+
+# Setup firewall before everything. docker creates it's own chain and the -X below will remove it
+# Do NOT use iptables-persistent because it's startup ordering conflicts with docker
+echo "=== Setting up firewall ==="
+# clear tables and set default policy
+iptables -F # flush all chains
+iptables -X # delete all chains
+# default policy for filter table
+iptables -P INPUT DROP
+iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject
+iptables -P OUTPUT ACCEPT
+
+# NOTE: keep these in sync with src/apps.js validatePortBindings
+# allow ssh, http, https, ping, dns
+iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
+iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
+iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
+iptables -A INPUT -p udp --sport 53 -j ACCEPT
+iptables -A INPUT -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+
+# loopback
+iptables -A INPUT -i lo -j ACCEPT
+iptables -A OUTPUT -o lo -j ACCEPT
+
+# prevent DoS
+# iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT
+
+# log dropped incoming. keep this at the end of all the rules
+iptables -N LOGGING # new chain
+iptables -A INPUT -j LOGGING # last rule in INPUT chain
+iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
+iptables -A LOGGING -j DROP
+
+echo "==== Install btrfs tools ==="
+apt-get -y install btrfs-tools
+
+echo "==== Install docker ===="
+# install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
+curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
+apt-get -y install aufs-tools
+chmod +x /usr/bin/docker
+groupadd docker
+cat > /etc/systemd/system/docker.socket <<EOF
+[Unit]
+Description=Docker Socket for the API
+PartOf=docker.service
+
+[Socket]
+ListenStream=/var/run/docker.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target
+EOF
+cat > /etc/systemd/system/docker.service <<EOF
+[Unit]
+Description=Docker Application Container Engine
+After=network.target docker.socket
+Requires=docker.socket
+
+[Service]
+ExecStart=/usr/bin/docker daemon -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+echo "=== Setup btrfs data ==="
+truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
+mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
+mkdir -p "${USER_DATA_DIR}"
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
+
+systemctl daemon-reload
+systemctl enable docker
+systemctl start docker
+
+# give docker sometime to start up and create iptables rules
+# those rules come in after docker has started, and we want to wait for them to be sure iptables-save has all of them
+sleep 10
+
+# Disable forwarding to metadata route from containers
+iptables -I FORWARD -d 169.254.169.254 -j DROP
+
+# ubuntu will restore iptables from this file automatically. this is here so that docker's chain is saved to this file
+mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4
+
+echo "=== Enable memory accounting =="
+sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
+
+# now add the user to the docker group
+usermod "${USER}" -a -G docker
+
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
+
+echo "==== Downloading docker images ===="
+images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage, Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+
+echo "Pulling images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+done
+
+echo "==== Install nginx ===="
+apt-get -y install nginx-full
+[[ "$(nginx -v 2>&1)" == *"nginx/1.10."* ]] || die "Expecting nginx version to be 1.10.x"
+
+echo "==== Install build-essential ===="
+apt-get -y install build-essential rcconf
+
+echo "==== Install mysql ===="
+debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
+debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
+apt-get -y install mysql-server-5.7
+[[ "$(mysqld --version 2>&1)" == *"5.7."* ]] || die "Expecting mysql version to be 5.7.x"
+
+echo "==== Install pwgen and swaks awscli ===="
+apt-get -y install pwgen swaks awscli
+
+echo "==== Install collectd ==="
+if ! apt-get install -y collectd collectd-utils; then
+    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+fi
+update-rc.d -f collectd remove
+
+# this simply makes it explicit that we run logrotate via cron. it's already part of base ubuntu
+echo "==== Install logrotate ==="
+apt-get install -y cron logrotate
+systemctl enable cron
+
+echo "=== Rebuilding npm packages ==="
+cd "${INSTALLER_SOURCE_DIR}" && npm install --production
+chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
+
+echo "==== Install installer systemd script ===="
+cat > /etc/systemd/system/cloudron-installer.service <<EOF
+[Unit]
+Description=Cloudron Installer
+; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
+BindsTo=systemd-journald.service
+
+[Service]
+Type=idle
+ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
+Environment="DEBUG=installer*,connect-lastmile"
+; kill any child (installer.sh) as well
+KillMode=control-group
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Restore iptables before docker
+echo "==== Install iptables-restore systemd script ===="
+cat > /etc/systemd/system/iptables-restore.service <<EOF
+[Unit]
+Description=IPTables Restore
+Before=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/sbin/iptables-restore /etc/iptables/rules.v4
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Allocate swap files
+# https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
+# On ubuntu ec2 we use cloud-init https://wiki.archlinux.org/index.php/Cloud-init
+echo "==== Install box-setup systemd script ===="
+cat > /etc/systemd/system/box-setup.service <<EOF
+[Unit]
+Description=Box Setup
+Before=docker.service collectd.service mysql.service
+After=do-resize.service cloud-init.service
+
+[Service]
+Type=oneshot
+ExecStart="${INSTALLER_SOURCE_DIR}/systemd/box-setup.sh"
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable cloudron-installer
+systemctl enable iptables-restore
+systemctl enable box-setup
+
+# Configure systemd
+sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
+    -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
+    -i /etc/systemd/journald.conf
+
+# When rotating logs, systemd kills journald too soon sometimes
+# See https://github.com/systemd/systemd/issues/1353 (this is upstream default)
+sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
+    -i /lib/systemd/system/systemd-journald.service
+
+sync
+
+# Configure time
+sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+timedatectl set-ntp 1
+timedatectl set-timezone UTC
+
+# Give user access to system logs
+apt-get -y install acl
+usermod -a -G systemd-journal ${USER}
+mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
+chown root:systemd-journal /var/log/journal
+systemctl restart systemd-journald
+setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
+
+echo "==== Install ssh ==="
+apt-get -y install openssh-server
+# https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
+sed -e 's/^#\?Port .*/Port 202/g' \
+    -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
+    -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
+    -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
+    -i /etc/ssh/sshd_config
+
+# required so we can connect to this machine since port 22 is blocked by iptables by now
+systemctl reload sshd
@@ -14,9 +14,9 @@ var appHealthMonitor = require('./src/apphealthmonitor.js'),
    async = require('async'),
    config = require('./src/config.js'),
    ldap = require('./src/ldap.js'),
-    simpleauth = require('./src/simpleauth.js'),
    oauthproxy = require('./src/oauthproxy.js'),
-    server = require('./src/server.js');
+    server = require('./src/server.js'),
+    simpleauth = require('./src/simpleauth.js');

 console.log();
 console.log('==========================================');
@@ -26,7 +26,6 @@ console.log();
 console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
 console.log(' Version:                        ', config.version());
 console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore token:                 ', config.token());
 console.log(' Appstore API server origin:     ', config.apiServerOrigin());
 console.log(' Appstore Web server origin:     ', config.webServerOrigin());
 console.log();
@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();
@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendFailureLogs(processName, { unit: processName });
+}
+
+main();
@@ -10,7 +10,7 @@ var ejs = require('gulp-ejs'),
    serve = require('gulp-serve'),
    sass = require('gulp-sass'),
    sourcemaps = require('gulp-sourcemaps'),
-    minifyCSS = require('gulp-minify-css'),
+    cssnano = require('gulp-cssnano'),
    autoprefixer = require('gulp-autoprefixer'),
    argv = require('yargs').argv;

@@ -22,6 +22,7 @@ gulp.task('3rdparty', function () {
        'webadmin/src/3rdparty/**/*.otf',
        'webadmin/src/3rdparty/**/*.eot',
        'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.gif',
        'webadmin/src/3rdparty/**/*.ttf',
        'webadmin/src/3rdparty/**/*.woff',
        'webadmin/src/3rdparty/**/*.woff2'
@@ -39,7 +40,7 @@ gulp.task('3rdparty', function () {
 // JavaScript
 // --------------

-gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
+gulp.task('js', ['js-index', 'js-setup', 'js-update'], function () {});

 var oauth = {
    clientId: argv.clientId || 'cid-webadmin',
@@ -80,14 +81,6 @@ gulp.task('js-setup', function () {
        .pipe(gulp.dest('webadmin/dist/js'));
 });

-gulp.task('js-error', function () {
-    gulp.src(['webadmin/src/js/error.js'])
-        .pipe(sourcemaps.init())
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'));
-});
-
 gulp.task('js-update', function () {
    gulp.src(['webadmin/src/js/update.js'])
        .pipe(sourcemaps.init())
@@ -102,7 +95,7 @@ gulp.task('js-update', function () {
 // HTML
 // --------------

-gulp.task('html', ['html-views', 'html-update'], function () {
+gulp.task('html', ['html-views', 'html-update', 'html-templates'], function () {
    return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
 });

@@ -114,6 +107,10 @@ gulp.task('html-views', function () {
    return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
 });

+gulp.task('html-templates', function () {
+    return gulp.src('webadmin/src/templates/**/*.html').pipe(gulp.dest('webadmin/dist/templates'));
+});
+
 // --------------
 // CSS
 // --------------
@@ -123,7 +120,7 @@ gulp.task('css', function () {
        .pipe(sourcemaps.init())
        .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
        .pipe(autoprefixer())
-        .pipe(minifyCSS())
+        .pipe(cssnano())
        .pipe(sourcemaps.write())
        .pipe(gulp.dest('webadmin/dist'))
        .pipe(gulp.dest('setup/splash/website'));
@@ -143,8 +140,8 @@ gulp.task('watch', ['default'], function () {
    gulp.watch(['webadmin/src/img/*'], ['images']);
    gulp.watch(['webadmin/src/**/*.html'], ['html']);
    gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
+    gulp.watch(['webadmin/src/templates/*.html'], ['html-templates']);
    gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
-    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
    gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
    gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
    gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);
@@ -0,0 +1,892 @@
+{
+  "name": "installer",
+  "version": "0.0.1",
+  "dependencies": {
+    "async": {
+      "version": "1.5.0",
+      "from": "https://registry.npmjs.org/async/-/async-1.5.0.tgz",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz"
+    },
+    "body-parser": {
+      "version": "1.14.1",
+      "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
+      "dependencies": {
+        "bytes": {
+          "version": "2.1.0",
+          "from": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz"
+        },
+        "content-type": {
+          "version": "1.0.1",
+          "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
+        },
+        "depd": {
+          "version": "1.1.0",
+          "from": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
+        },
+        "http-errors": {
+          "version": "1.3.1",
+          "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+          "dependencies": {
+            "inherits": {
+              "version": "2.0.1",
+              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
+              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+            },
+            "statuses": {
+              "version": "1.2.1",
+              "from": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz",
+              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+            }
+          }
+        },
+        "iconv-lite": {
+          "version": "0.4.12",
+          "from": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "qs": {
+          "version": "5.1.0",
+          "from": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz"
+        },
+        "raw-body": {
+          "version": "2.1.4",
+          "from": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
+          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
+          "dependencies": {
+            "unpipe": {
+              "version": "1.0.0",
+              "from": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+            }
+          }
+        },
+        "type-is": {
+          "version": "1.6.9",
+          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "dependencies": {
+            "media-typer": {
+              "version": "0.3.0",
+              "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "connect-lastmile": {
+      "version": "0.0.13",
+      "from": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
+      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
+      "dependencies": {
+        "debug": {
+          "version": "2.1.3",
+          "from": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
+          "dependencies": {
+            "ms": {
+              "version": "0.7.0",
+              "from": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz",
+              "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
+            }
+          }
+        }
+      }
+    },
+    "debug": {
+      "version": "2.2.0",
+      "from": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "dependencies": {
+        "ms": {
+          "version": "0.7.1",
+          "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        }
+      }
+    },
+    "express": {
+      "version": "4.13.3",
+      "from": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
+      "dependencies": {
+        "accepts": {
+          "version": "1.2.13",
+          "from": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
+          "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
+          "dependencies": {
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            },
+            "negotiator": {
+              "version": "0.5.3",
+              "from": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz",
+              "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
+            }
+          }
+        },
+        "array-flatten": {
+          "version": "1.1.1",
+          "from": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+          "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
+        },
+        "content-disposition": {
+          "version": "0.5.0",
+          "from": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz",
+          "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz"
+        },
+        "content-type": {
+          "version": "1.0.1",
+          "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
+        },
+        "cookie": {
+          "version": "0.1.3",
+          "from": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz"
+        },
+        "cookie-signature": {
+          "version": "1.0.6",
+          "from": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
+          "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
+        },
+        "depd": {
+          "version": "1.0.1",
+          "from": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz",
+          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+        },
+        "escape-html": {
+          "version": "1.0.2",
+          "from": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz",
+          "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz"
+        },
+        "etag": {
+          "version": "1.7.0",
+          "from": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz",
+          "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
+        },
+        "finalhandler": {
+          "version": "0.4.0",
+          "from": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
+          "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
+          "dependencies": {
+            "unpipe": {
+              "version": "1.0.0",
+              "from": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+            }
+          }
+        },
+        "fresh": {
+          "version": "0.3.0",
+          "from": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
+        },
+        "merge-descriptors": {
+          "version": "1.0.0",
+          "from": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
+        },
+        "methods": {
+          "version": "1.1.1",
+          "from": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "parseurl": {
+          "version": "1.3.0",
+          "from": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
+        },
+        "path-to-regexp": {
+          "version": "0.1.7",
+          "from": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
+          "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
+        },
+        "proxy-addr": {
+          "version": "1.0.8",
+          "from": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
+          "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
+          "dependencies": {
+            "forwarded": {
+              "version": "0.1.0",
+              "from": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz",
+              "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz"
+            },
+            "ipaddr.js": {
+              "version": "1.0.1",
+              "from": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz",
+              "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz"
+            }
+          }
+        },
+        "qs": {
+          "version": "4.0.0",
+          "from": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
+        },
+        "range-parser": {
+          "version": "1.0.3",
+          "from": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz",
+          "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
+        },
+        "send": {
+          "version": "0.13.0",
+          "from": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
+          "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
+          "dependencies": {
+            "destroy": {
+              "version": "1.0.3",
+              "from": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz",
+              "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz"
+            },
+            "http-errors": {
+              "version": "1.3.1",
+              "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+              "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+              "dependencies": {
+                "inherits": {
+                  "version": "2.0.1",
+                  "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
+                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.3.4",
+              "from": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz",
+              "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
+            },
+            "ms": {
+              "version": "0.7.1",
+              "from": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
+              "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+            },
+            "statuses": {
+              "version": "1.2.1",
+              "from": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz",
+              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+            }
+          }
+        },
+        "serve-static": {
+          "version": "1.10.0",
+          "from": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz",
+          "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
+        },
+        "type-is": {
+          "version": "1.6.9",
+          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "dependencies": {
+            "media-typer": {
+              "version": "0.3.0",
+              "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            }
+          }
+        },
+        "utils-merge": {
+          "version": "1.0.0",
+          "from": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz",
+          "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
+        },
+        "vary": {
+          "version": "1.0.1",
+          "from": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
+        }
+      }
+    },
+    "json": {
+      "version": "9.0.3",
+      "from": "https://registry.npmjs.org/json/-/json-9.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz"
+    },
+    "morgan": {
+      "version": "1.6.1",
+      "from": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
+      "dependencies": {
+        "basic-auth": {
+          "version": "1.0.3",
+          "from": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz",
+          "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz"
+        },
+        "depd": {
+          "version": "1.0.1",
+          "from": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz",
+          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "on-headers": {
+          "version": "1.0.1",
+          "from": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz"
+        }
+      }
+    },
+    "proxy-middleware": {
+      "version": "0.15.0",
+      "from": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz",
+      "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz"
+    },
+    "request": {
+      "version": "2.72.0",
+      "from": "request@*",
+      "resolved": "https://registry.npmjs.org/request/-/request-2.72.0.tgz",
+      "dependencies": {
+        "aws-sign2": {
+          "version": "0.6.0",
+          "from": "aws-sign2@>=0.6.0 <0.7.0",
+          "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
+        },
+        "aws4": {
+          "version": "1.4.1",
+          "from": "aws4@>=1.2.1 <2.0.0",
+          "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.4.1.tgz"
+        },
+        "bl": {
+          "version": "1.1.2",
+          "from": "bl@>=1.1.2 <1.2.0",
+          "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz",
+          "dependencies": {
+            "readable-stream": {
+              "version": "2.0.6",
+              "from": "readable-stream@>=2.0.5 <2.1.0",
+              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
+              "dependencies": {
+                "core-util-is": {
+                  "version": "1.0.2",
+                  "from": "core-util-is@>=1.0.0 <1.1.0",
+                  "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
+                },
+                "inherits": {
+                  "version": "2.0.1",
+                  "from": "inherits@>=2.0.1 <2.1.0",
+                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+                },
+                "isarray": {
+                  "version": "1.0.0",
+                  "from": "isarray@>=1.0.0 <1.1.0",
+                  "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
+                },
+                "process-nextick-args": {
+                  "version": "1.0.7",
+                  "from": "process-nextick-args@>=1.0.6 <1.1.0",
+                  "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz"
+                },
+                "string_decoder": {
+                  "version": "0.10.31",
+                  "from": "string_decoder@>=0.10.0 <0.11.0",
+                  "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+                },
+                "util-deprecate": {
+                  "version": "1.0.2",
+                  "from": "util-deprecate@>=1.0.1 <1.1.0",
+                  "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
+                }
+              }
+            }
+          }
+        },
+        "caseless": {
+          "version": "0.11.0",
+          "from": "caseless@>=0.11.0 <0.12.0",
+          "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz"
+        },
+        "combined-stream": {
+          "version": "1.0.5",
+          "from": "combined-stream@>=1.0.5 <1.1.0",
+          "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
+          "dependencies": {
+            "delayed-stream": {
+              "version": "1.0.0",
+              "from": "delayed-stream@>=1.0.0 <1.1.0",
+              "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz"
+            }
+          }
+        },
+        "extend": {
+          "version": "3.0.0",
+          "from": "extend@>=3.0.0 <3.1.0",
+          "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
+        },
+        "forever-agent": {
+          "version": "0.6.1",
+          "from": "forever-agent@>=0.6.1 <0.7.0",
+          "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
+        },
+        "form-data": {
+          "version": "1.0.0-rc4",
+          "from": "form-data@>=1.0.0-rc3 <1.1.0",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc4.tgz",
+          "dependencies": {
+            "async": {
+              "version": "1.5.2",
+              "from": "async@>=1.5.2 <2.0.0",
+              "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
+            }
+          }
+        },
+        "har-validator": {
+          "version": "2.0.6",
+          "from": "har-validator@>=2.0.6 <2.1.0",
+          "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz",
+          "dependencies": {
+            "chalk": {
+              "version": "1.1.3",
+              "from": "chalk@>=1.1.1 <2.0.0",
+              "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+              "dependencies": {
+                "ansi-styles": {
+                  "version": "2.2.1",
+                  "from": "ansi-styles@>=2.2.1 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz"
+                },
+                "escape-string-regexp": {
+                  "version": "1.0.5",
+                  "from": "escape-string-regexp@>=1.0.2 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
+                },
+                "has-ansi": {
+                  "version": "2.0.0",
+                  "from": "has-ansi@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
+                  "dependencies": {
+                    "ansi-regex": {
+                      "version": "2.0.0",
+                      "from": "ansi-regex@>=2.0.0 <3.0.0",
+                      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
+                    }
+                  }
+                },
+                "strip-ansi": {
+                  "version": "3.0.1",
+                  "from": "strip-ansi@>=3.0.0 <4.0.0",
+                  "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+                  "dependencies": {
+                    "ansi-regex": {
+                      "version": "2.0.0",
+                      "from": "ansi-regex@>=2.0.0 <3.0.0",
+                      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
+                    }
+                  }
+                },
+                "supports-color": {
+                  "version": "2.0.0",
+                  "from": "supports-color@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz"
+                }
+              }
+            },
+            "commander": {
+              "version": "2.9.0",
+              "from": "commander@>=2.9.0 <3.0.0",
+              "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz",
+              "dependencies": {
+                "graceful-readlink": {
+                  "version": "1.0.1",
+                  "from": "graceful-readlink@>=1.0.0",
+                  "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz"
+                }
+              }
+            },
+            "is-my-json-valid": {
+              "version": "2.13.1",
+              "from": "is-my-json-valid@>=2.12.4 <3.0.0",
+              "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz",
+              "dependencies": {
+                "generate-function": {
+                  "version": "2.0.0",
+                  "from": "generate-function@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz"
+                },
+                "generate-object-property": {
+                  "version": "1.2.0",
+                  "from": "generate-object-property@>=1.1.0 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz",
+                  "dependencies": {
+                    "is-property": {
+                      "version": "1.0.2",
+                      "from": "is-property@>=1.0.0 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz"
+                    }
+                  }
+                },
+                "jsonpointer": {
+                  "version": "2.0.0",
+                  "from": "jsonpointer@2.0.0",
+                  "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz"
+                },
+                "xtend": {
+                  "version": "4.0.1",
+                  "from": "xtend@>=4.0.0 <5.0.0",
+                  "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz"
+                }
+              }
+            },
+            "pinkie-promise": {
+              "version": "2.0.1",
+              "from": "pinkie-promise@>=2.0.0 <3.0.0",
+              "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
+              "dependencies": {
+                "pinkie": {
+                  "version": "2.0.4",
+                  "from": "pinkie@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
+                }
+              }
+            }
+          }
+        },
+        "hawk": {
+          "version": "3.1.3",
+          "from": "hawk@>=3.1.3 <3.2.0",
+          "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
+          "dependencies": {
+            "hoek": {
+              "version": "2.16.3",
+              "from": "hoek@>=2.0.0 <3.0.0",
+              "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
+            },
+            "boom": {
+              "version": "2.10.1",
+              "from": "boom@>=2.0.0 <3.0.0",
+              "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
+            },
+            "cryptiles": {
+              "version": "2.0.5",
+              "from": "cryptiles@>=2.0.0 <3.0.0",
+              "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
+            },
+            "sntp": {
+              "version": "1.0.9",
+              "from": "sntp@>=1.0.0 <2.0.0",
+              "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
+            }
+          }
+        },
+        "http-signature": {
+          "version": "1.1.1",
+          "from": "http-signature@>=1.1.0 <1.2.0",
+          "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
+          "dependencies": {
+            "assert-plus": {
+              "version": "0.2.0",
+              "from": "assert-plus@>=0.2.0 <0.3.0",
+              "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
+            },
+            "jsprim": {
+              "version": "1.2.2",
+              "from": "jsprim@>=1.2.2 <2.0.0",
+              "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz",
+              "dependencies": {
+                "extsprintf": {
+                  "version": "1.0.2",
+                  "from": "extsprintf@1.0.2",
+                  "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz"
+                },
+                "json-schema": {
+                  "version": "0.2.2",
+                  "from": "json-schema@0.2.2",
+                  "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz"
+                },
+                "verror": {
+                  "version": "1.3.6",
+                  "from": "verror@1.3.6",
+                  "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz"
+                }
+              }
+            },
+            "sshpk": {
+              "version": "1.8.3",
+              "from": "sshpk@>=1.7.0 <2.0.0",
+              "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.8.3.tgz",
+              "dependencies": {
+                "asn1": {
+                  "version": "0.2.3",
+                  "from": "asn1@>=0.2.3 <0.3.0",
+                  "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz"
+                },
+                "assert-plus": {
+                  "version": "1.0.0",
+                  "from": "assert-plus@>=1.0.0 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+                },
+                "dashdash": {
+                  "version": "1.14.0",
+                  "from": "dashdash@>=1.12.0 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.0.tgz"
+                },
+                "getpass": {
+                  "version": "0.1.6",
+                  "from": "getpass@>=0.1.1 <0.2.0",
+                  "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz"
+                },
+                "jsbn": {
+                  "version": "0.1.0",
+                  "from": "jsbn@>=0.1.0 <0.2.0",
+                  "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz"
+                },
+                "tweetnacl": {
+                  "version": "0.13.3",
+                  "from": "tweetnacl@>=0.13.0 <0.14.0",
+                  "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.13.3.tgz"
+                },
+                "jodid25519": {
+                  "version": "1.0.2",
+                  "from": "jodid25519@>=1.0.0 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz"
+                },
+                "ecc-jsbn": {
+                  "version": "0.1.1",
+                  "from": "ecc-jsbn@>=0.1.1 <0.2.0",
+                  "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz"
+                }
+              }
+            }
+          }
+        },
+        "is-typedarray": {
+          "version": "1.0.0",
+          "from": "is-typedarray@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz"
+        },
+        "isstream": {
+          "version": "0.1.2",
+          "from": "isstream@>=0.1.2 <0.2.0",
+          "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
+        },
+        "json-stringify-safe": {
+          "version": "5.0.1",
+          "from": "json-stringify-safe@>=5.0.1 <5.1.0",
+          "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
+        },
+        "mime-types": {
+          "version": "2.1.11",
+          "from": "mime-types@>=2.1.7 <2.2.0",
+          "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.11.tgz",
+          "dependencies": {
+            "mime-db": {
+              "version": "1.23.0",
+              "from": "mime-db@>=1.23.0 <1.24.0",
+              "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.23.0.tgz"
+            }
+          }
+        },
+        "node-uuid": {
+          "version": "1.4.7",
+          "from": "node-uuid@>=1.4.7 <1.5.0",
+          "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz"
+        },
+        "oauth-sign": {
+          "version": "0.8.2",
+          "from": "oauth-sign@>=0.8.1 <0.9.0",
+          "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz"
+        },
+        "qs": {
+          "version": "6.1.0",
+          "from": "qs@>=6.1.0 <6.2.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.1.0.tgz"
+        },
+        "stringstream": {
+          "version": "0.0.5",
+          "from": "stringstream@>=0.0.4 <0.1.0",
+          "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
+        },
+        "tough-cookie": {
+          "version": "2.2.2",
+          "from": "tough-cookie@>=2.2.0 <2.3.0",
+          "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz"
+        },
+        "tunnel-agent": {
+          "version": "0.4.3",
+          "from": "tunnel-agent@>=0.4.1 <0.5.0",
+          "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz"
+        }
+      }
+    },
+    "safetydance": {
+      "version": "0.0.19",
+      "from": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz",
+      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
+    },
+    "semver": {
+      "version": "5.1.0",
+      "from": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
+    },
+    "superagent": {
+      "version": "0.21.0",
+      "from": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
+      "dependencies": {
+        "qs": {
+          "version": "1.2.0",
+          "from": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz"
+        },
+        "formidable": {
+          "version": "1.0.14",
+          "from": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz",
+          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz"
+        },
+        "mime": {
+          "version": "1.2.11",
+          "from": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
+        },
+        "component-emitter": {
+          "version": "1.1.2",
+          "from": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz",
+          "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz"
+        },
+        "methods": {
+          "version": "1.0.1",
+          "from": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz"
+        },
+        "cookiejar": {
+          "version": "2.0.1",
+          "from": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz"
+        },
+        "reduce-component": {
+          "version": "1.0.1",
+          "from": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz",
+          "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
+        },
+        "extend": {
+          "version": "1.2.1",
+          "from": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz",
+          "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz"
+        },
+        "form-data": {
+          "version": "0.1.3",
+          "from": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
+          "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
+          "dependencies": {
+            "combined-stream": {
+              "version": "0.0.7",
+              "from": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
+              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
+              "dependencies": {
+                "delayed-stream": {
+                  "version": "0.0.5",
+                  "from": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz",
+                  "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
+                }
+              }
+            },
+            "async": {
+              "version": "0.9.2",
+              "from": "https://registry.npmjs.org/async/-/async-0.9.2.tgz",
+              "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
+            }
+          }
+        },
+        "readable-stream": {
+          "version": "1.0.27-1",
+          "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
+          "dependencies": {
+            "core-util-is": {
+              "version": "1.0.1",
+              "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz",
+              "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
+            },
+            "isarray": {
+              "version": "0.0.1",
+              "from": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
+              "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
+            },
+            "string_decoder": {
+              "version": "0.10.31",
+              "from": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
+              "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+            },
+            "inherits": {
+              "version": "2.0.1",
+              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
+              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+            }
+          }
+        }
+      }
+    }
+  }
+}
@@ -0,0 +1,48 @@
+{
+  "name": "installer",
+  "description": "Cloudron Installer",
+  "version": "0.0.1",
+  "private": "true",
+  "author": {
+    "name": "Cloudron authors"
+  },
+  "repository": {
+    "type": "git"
+  },
+  "engines": [
+    "node >=4.0.0 <=4.1.1"
+  ],
+  "dependencies": {
+    "async": "^1.5.0",
+    "body-parser": "^1.12.0",
+    "connect-lastmile": "0.0.13",
+    "debug": "^2.1.1",
+    "express": "^4.11.2",
+    "json": "^9.0.3",
+    "morgan": "^1.5.1",
+    "proxy-middleware": "^0.15.0",
+    "request": "^2.72.0",
+    "safetydance": "0.0.19",
+    "semver": "^5.1.0",
+    "superagent": "^0.21.0"
+  },
+  "devDependencies": {
+    "colors": "^1.1.2",
+    "commander": "^2.8.1",
+    "expect.js": "^0.3.1",
+    "istanbul": "^0.3.5",
+    "lodash": "^3.2.0",
+    "mocha": "^2.1.0",
+    "nock": "^0.59.1",
+    "sleep": "^3.0.0",
+    "superagent-sync": "^0.2.0",
+    "supererror": "^0.7.0",
+    "yesno": "0.0.1"
+  },
+  "scripts": {
+    "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test",
+    "precommit": "/bin/true",
+    "prepush": "npm test",
+    "postmerge": "/bin/true"
+  }
+}
@@ -0,0 +1,112 @@
+/* jslint node: true */
+
+'use strict';
+
+var assert = require('assert'),
+    child_process = require('child_process'),
+    debug = require('debug')('installer:installer'),
+    path = require('path'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    superagent = require('superagent'),
+    util = require('util');
+
+exports = module.exports = {
+    InstallerError: InstallerError,
+
+    provision: provision,
+
+    _ensureVersion: ensureVersion
+};
+
+var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'),
+    SUDO = '/usr/bin/sudo';
+
+function InstallerError(reason, info) {
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info);
+}
+util.inherits(InstallerError, Error);
+InstallerError.INTERNAL_ERROR = 1;
+InstallerError.ALREADY_PROVISIONED = 2;
+
+// system until file has KillMode=control-group to bring down child processes
+function spawn(tag, cmd, args, callback) {
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof cmd, 'string');
+    assert(util.isArray(args));
+    assert.strictEqual(typeof callback, 'function');
+
+    var cp = child_process.spawn(cmd, args, { timeout: 0 });
+    cp.stdout.setEncoding('utf8');
+    cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); });
+    cp.stderr.setEncoding('utf8');
+    cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); });
+
+    cp.on('error', function (error) {
+        debug('%s : child process errored %s', tag, error.message);
+        callback(error);
+    });
+
+    cp.on('exit', function (code, signal) {
+        debug('%s : child process exited. code: %d signal: %d', tag, code, signal);
+        if (signal) return callback(new Error('Exited with signal ' + signal));
+        if (code !== 0) return callback(new Error('Exited with code ' + code));
+
+        callback(null);
+    });
+}
+
+function ensureVersion(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!args.data || !args.data.boxVersionsUrl) return callback(new Error('No boxVersionsUrl specified'));
+
+    if (args.sourceTarballUrl) return callback(null, args);
+
+    superagent.get(args.data.boxVersionsUrl).end(function (error, result) {
+        if (error && !error.response) return callback(error);
+        if (result.statusCode !== 200) return callback(new Error(util.format('Bad status: %s %s', result.statusCode, result.text)));
+
+        var versions = safe.JSON.parse(result.text);
+
+        if (!versions || typeof versions !== 'object') return callback(new Error('versions is not in valid format:' + safe.error));
+
+        var latestVersion = Object.keys(versions).sort(semver.compare).pop();
+        debug('ensureVersion: Latest version is %s etag:%s', latestVersion, result.header['etag']);
+
+        if (!versions[latestVersion]) return callback(new Error('No version available'));
+        if (!versions[latestVersion].sourceTarballUrl) return callback(new Error('No sourceTarballUrl specified'));
+
+        args.sourceTarballUrl = versions[latestVersion].sourceTarballUrl;
+        args.data.version = latestVersion;
+
+        callback(null, args);
+    });
+}
+
+function provision(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (process.env.NODE_ENV === 'test') return callback(null);
+
+    ensureVersion(args, function (error, result) {
+        if (error) return callback(error);
+
+        var pargs = [ INSTALLER_CMD ];
+        pargs.push('--sourcetarballurl', result.sourceTarballUrl);
+        pargs.push('--data', JSON.stringify(result.data));
+
+        debug('provision: calling with args %j', pargs);
+
+        // sudo is required for update()
+        spawn('provision', SUDO, pargs, callback);
+    });
+}
+
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly BOX_SRC_DIR=/home/yellowtent/box
+readonly DATA_DIR=/home/yellowtent/data
+
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly json="${script_dir}/../../node_modules/.bin/json"
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
+
+readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
+
+# create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@
+(echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh
+chmod +x /home/yellowtent/provision.sh
+
+arg_source_tarball_url=""
+arg_data=""
+
+args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --sourcetarballurl) arg_source_tarball_url="$2";;
+    --data) arg_data="$2";;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+
+    shift 2
+done
+
+box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
+echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}"
+
+while true; do
+    if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi
+    echo "Failed to download source tarball, trying again"
+    sleep 5
+done
+while true; do
+    # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
+    if cd "${box_src_tmp_dir}" && npm rebuild; then break; fi
+    echo "Failed to rebuild, trying again"
+    sleep 5
+done
+
+if [[ "${is_update}" == "yes" ]]; then
+    echo "Setting up update splash screen"
+    "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code
+    ${BOX_SRC_DIR}/setup/stop.sh # stop the old code
+fi
+
+# switch the codes
+rm -rf "${BOX_SRC_DIR}"
+mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}"
+chown -R yellowtent.yellowtent "${BOX_SRC_DIR}"
+
+# create a start file for testing. %q escapes args
+(echo -e "#!/bin/bash\n"; printf "%q " "${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}") > /home/yellowtent/setup_start.sh
+chmod +x /home/yellowtent/setup_start.sh
+
+echo "Calling box setup script"
+"${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}"
+
@@ -0,0 +1,170 @@
+#!/usr/bin/env node
+
+/* jslint node: true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    debug = require('debug')('installer:server'),
+    express = require('express'),
+    fs = require('fs'),
+    http = require('http'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    installer = require('./installer.js'),
+    json = require('body-parser').json,
+    lastMile = require('connect-lastmile'),
+    morgan = require('morgan'),
+    request = require('request'),
+    superagent = require('superagent');
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var PROVISION_CONFIG_FILE = '/root/provision.json';
+var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
+
+var gHttpServer = null; // update server; used for updates
+
+function provisionDigitalOcean(callback) {
+    superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
+        if (error || result.statusCode !== 200) {
+            console.error('Error getting metadata', error);
+            return callback(new Error('Error getting metadata'));
+        }
+
+        callback(null, JSON.parse(result.body.user_data));
+    });
+}
+
+function provisionEC2(callback) {
+    // need to use request, since octet-stream data
+    request('http://169.254.169.254/latest/user-data', function (error, response, body) {
+        if (error || response.statusCode !== 200) {
+            console.error('Error getting metadata', error);
+            return callback(new Error('Error getting metadata'));
+        }
+
+        callback(null, JSON.parse(body));
+    });
+}
+
+function provision(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    // try first digitalocean, then ec2
+    provisionDigitalOcean(function (error, userData) {
+        if (!error) return installer.provision(userData, callback);
+
+        provisionEC2(function (error, userData) {
+            if (!error) return installer.provision(userData, callback);
+
+            console.error('Unable to get meta data', error);
+
+            callback(new Error('Error getting metadata'));
+        });
+    });
+}
+
+function provisionLocal(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    if (!fs.existsSync(PROVISION_CONFIG_FILE)) {
+        console.error('No provisioning data found at %s', PROVISION_CONFIG_FILE);
+        return callback(new Error('No provisioning data found'));
+    }
+
+    var userData = require(PROVISION_CONFIG_FILE);
+
+    installer.provision(userData, callback);
+}
+
+function update(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.sourceTarballUrl || typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided'));
+    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
+
+    debug('provision: received from box %j', req.body);
+
+    installer.provision(req.body, function (error) {
+        if (error) console.error(error);
+    });
+
+    next(new HttpSuccess(202, { }));
+}
+
+function startUpdateServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting update server');
+
+    var app = express();
+
+    var router = new express.Router();
+
+    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
+
+    app.use(json({ strict: true }))
+       .use(router)
+       .use(lastMile());
+
+    router.post('/api/v1/installer/update', update);
+
+    gHttpServer = http.createServer(app);
+    gHttpServer.on('error', console.error);
+
+    gHttpServer.listen(2020, '127.0.0.1', callback);
+}
+
+function stopUpdateServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Stopping update server');
+
+    if (!gHttpServer) return callback(null);
+
+    gHttpServer.close(callback);
+    gHttpServer = null;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var actions;
+
+    if (process.env.PROVISION === 'local') {
+        debug('Starting Installer in selfhost mode');
+
+        actions = [
+            startUpdateServer,
+            provisionLocal
+        ];
+    } else {    // current fallback, should be 'digitalocean' eventually, see initializeBaseUbuntuImage.sh
+        debug('Starting Installer in managed mode');
+
+        actions = [
+            startUpdateServer,
+            provision
+        ];
+    }
+
+    async.series(actions, callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        stopUpdateServer
+    ], callback);
+}
+
+if (require.main === module) {
+    start(function (error) {
+        if (error) console.error(error);
+    });
+}
@@ -0,0 +1,179 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var expect = require('expect.js'),
+    fs = require('fs'),
+    path = require('path'),
+    nock = require('nock'),
+    os = require('os'),
+    request = require('superagent'),
+    server = require('../server.js'),
+    installer = require('../installer.js'),
+    _ = require('lodash');
+
+var EXTERNAL_SERVER_URL = 'https://localhost:4443';
+var INTERNAL_SERVER_URL = 'http://localhost:2020';
+var APPSERVER_ORIGIN = 'http://appserver';
+var FQDN = os.hostname();
+
+describe('Server', function () {
+    this.timeout(5000);
+
+    before(function (done) {
+        var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string
+        var scope = nock('http://169.254.169.254')
+            .persist()
+            .get('/metadata/v1.json')
+            .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' });
+        done();
+    });
+
+    after(function (done) {
+        nock.cleanAll();
+        done();
+    });
+
+    describe('starts and stop', function () {
+        it('starts', function (done) {
+            server.start(done);
+        });
+
+        it('stops', function (done) {
+            server.stop(done);
+        });
+    });
+
+    describe('update (internal server)', function () {
+        before(function (done) {
+            server.start(done);
+        });
+        after(function (done) {
+            server.stop(done);
+        });
+
+        it('does not respond to provision', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        it('does not respond to restore', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        var data = {
+            sourceTarballUrl: "https://foo.tar.gz",
+
+            data: {
+                token: 'sometoken',
+                apiServerOrigin: APPSERVER_ORIGIN,
+                webServerOrigin: 'https://somethingelse.com',
+                fqdn: 'www.something.com',
+                tlsKey: 'key',
+                tlsCert: 'cert',
+                boxVersionsUrl: 'https://versions.json',
+                version: '0.1'
+            }
+        };
+
+        Object.keys(data).forEach(function (key) {
+            it('fails due to missing ' + key, function (done) {
+                var dataCopy = _.merge({ }, data);
+                delete dataCopy[key];
+
+                request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+            });
+        });
+
+        it('succeeds', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+    });
+
+    describe('ensureVersion', function () {
+        before(function () {
+            process.env.NODE_ENV = undefined;
+        });
+
+        after(function () {
+            process.env.NODE_ENV = 'test';
+        });
+
+        it ('fails without data', function (done) {
+            installer._ensureVersion({}, function (error) {
+                expect(error).to.be.an(Error);
+                done();
+            });
+        });
+
+        it ('fails without boxVersionsUrl', function (done) {
+            installer._ensureVersion({ data: {}}, function (error) {
+                expect(error).to.be.an(Error);
+                done();
+            });
+        });
+
+        it ('succeeds with sourceTarballUrl', function (done) {
+            var data = {
+                sourceTarballUrl: 'sometarballurl',
+                data: {
+                    boxVersionsUrl: 'http://foobar/versions.json'
+                }
+            };
+
+            installer._ensureVersion(data, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result).to.eql(data);
+                done();
+            });
+        });
+
+        it ('succeeds without sourceTarballUrl', function (done) {
+            var versions = {
+                '0.1.0': {
+                    sourceTarballUrl: 'sometarballurl1'
+                },
+                '0.2.0': {
+                    sourceTarballUrl: 'sometarballurl2'
+                }
+            };
+
+            var scope = nock('http://foobar')
+                .get('/versions.json')
+                .reply(200, JSON.stringify(versions), { 'Content-Type': 'application/json' });
+
+            var data = {
+                data: {
+                    boxVersionsUrl: 'http://foobar/versions.json'
+                }
+            };
+
+            installer._ensureVersion(data, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result.sourceTarballUrl).to.equal(versions['0.2.0'].sourceTarballUrl);
+                expect(result.data.boxVersionsUrl).to.equal(data.data.boxVersionsUrl);
+                done();
+            });
+        });
+    });
+});
+
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly USER_HOME="/home/yellowtent"
+readonly APPS_SWAP_FILE="/apps.swap"
+readonly USER_DATA_FILE="/root/user_data.img"
+readonly USER_DATA_DIR="/home/yellowtent/data"
+
+# detect device
+if [[ -b "/dev/vda1" ]]; then
+    disk_device="/dev/vda1"
+fi
+
+if [[ -b "/dev/xvda1" ]]; then
+    disk_device="/dev/xvda1"
+fi
+
+# allow root access over ssh
+sed -e 's/.* \(ssh-rsa.*\)/\1/' -i /root/.ssh/authorized_keys
+
+# all sizes are in mb
+readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
+readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
+readonly app_count=$((${physical_memory} / 200)) # estimated app count
+readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
+readonly disk_size=$((disk_size_gb * 1024))
+readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
+readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
+
+echo "Disk device: ${disk_device}"
+echo "Physical memory: ${physical_memory}"
+echo "Estimated app count: ${app_count}"
+echo "Disk size: ${disk_size}"
+
+# Allocate swap for general app usage
+if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
+    echo "Creating Apps swap file of size ${swap_size}M"
+    fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
+    chmod 600 "${APPS_SWAP_FILE}"
+    mkswap "${APPS_SWAP_FILE}"
+    swapon "${APPS_SWAP_FILE}"
+    echo "${APPS_SWAP_FILE}  none  swap  sw  0 0" >> /etc/fstab
+else
+    echo "Apps Swap file already exists"
+fi
+
+echo "Resizing data volume"
+home_data_size=$((disk_size - system_size - swap_size - ext4_reserved))
+echo "Resizing up btrfs user data to size ${home_data_size}M"
+umount "${USER_DATA_DIR}" || true
+# Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data
+# fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
+truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
+btrfs filesystem resize max "${USER_DATA_DIR}"
+
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN displayName VARCHAR(512) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN displayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN memoryLimit BIGINT DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN memoryLimit', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,21 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE groups(" +
+                "id VARCHAR(128) NOT NULL UNIQUE," +
+                "name VARCHAR(128) NOT NULL UNIQUE," +
+                "PRIMARY KEY(id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,22 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = "CREATE TABLE IF NOT EXISTS groupMembers(" +
+    			"groupId VARCHAR(128) NOT NULL," +
+    			"userId VARCHAR(128) NOT NULL," +
+    			"FOREIGN KEY(groupId) REFERENCES groups(id)," +
+    			"FOREIGN KEY(userId) REFERENCES users(id));";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groupMembers', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,30 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var async = require('async');
+
+var ADMIN_GROUP_ID = 'admin'; // see groups.js
+
+exports.up = function(db, callback) {
+	async.series([
+		db.runSql.bind(db, 'START TRANSACTION;'),
+		db.runSql.bind(db, 'INSERT INTO groups (id, name) VALUES (?, ?)', [ ADMIN_GROUP_ID, 'admin' ]),
+		function migrateAdminFlag(done) {
+			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
+				if (error) return done(error);
+
+				console.dir(results);
+
+				async.eachSeries(results, function (r, next) {
+					db.runSql('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ ADMIN_GROUP_ID, r.id ], next);
+				}, done);
+			});
+		},
+		db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+		db.runSql.bind(db, 'COMMIT')
+	], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};
@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE backups(" +
+            "filename VARCHAR(128) NOT NULL," +
+            "creationTime TIMESTAMP," +
+            "version VARCHAR(128) NOT NULL," +
+            "type VARCHAR(16) NOT NULL," +
+            "dependsOn VARCHAR(4096)," +
+            "state VARCHAR(16) NOT NULL," +
+            "PRIMARY KEY (filename))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE backups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
@@ -0,0 +1,17 @@
+var dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE filename id VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE id filename VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,17 @@
+'use strict';
+
+var dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN altDomain VARCHAR(256)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN altDomain', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,24 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE eventlog(" +
+            "id VARCHAR(128) NOT NULL," +
+            "source JSON," +
+            "creationTime TIMESTAMP," +
+            "action VARCHAR(128) NOT NULL," +
+            "data JSON," +
+            "PRIMARY KEY (id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE eventlog', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN showTutorial BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN showTutorial', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,24 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};
@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN xFrameOptions VARCHAR(512)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN xFrameOptions', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -11,21 +11,34 @@

 CREATE TABLE IF NOT EXISTS users(
    id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(254) NOT NULL UNIQUE,
+    username VARCHAR(254) UNIQUE,
    email VARCHAR(254) NOT NULL UNIQUE,
    password VARCHAR(1024) NOT NULL,
    salt VARCHAR(512) NOT NULL,
    createdAt VARCHAR(512) NOT NULL,
    modifiedAt VARCHAR(512) NOT NULL,
    admin INTEGER NOT NULL,
+    displayName VARCHAR(512) DEFAULT '',
+    showTutorial BOOLEAN DEFAULT 0,
    PRIMARY KEY(id));

+CREATE TABLE IF NOT EXISTS groups(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS groupMembers(
+    groupId VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    FOREIGN KEY(groupId) REFERENCES groups(id),
+    FOREIGN KEY(userId) REFERENCES users(id));
+
 CREATE TABLE IF NOT EXISTS tokens(
    accessToken VARCHAR(128) NOT NULL UNIQUE,
    identifier VARCHAR(128) NOT NULL,
    clientId VARCHAR(128),
    scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires BIGINT NOT NULL, // FIXME: make this a timestamp
    PRIMARY KEY(accessToken));

 CREATE TABLE IF NOT EXISTS clients(
@@ -41,7 +54,7 @@ CREATE TABLE IF NOT EXISTS apps(
    id VARCHAR(128) NOT NULL UNIQUE,
    appStoreId VARCHAR(128) NOT NULL,
    installationState VARCHAR(512) NOT NULL,
-    installationProgress VARCHAR(512),
+    installationProgress TEXT,
    runState VARCHAR(512),
    health VARCHAR(128),
    containerId VARCHAR(128),
@@ -49,14 +62,16 @@ CREATE TABLE IF NOT EXISTS apps(
    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
    location VARCHAR(128) NOT NULL UNIQUE,
    dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
    oauthProxy BOOLEAN DEFAULT 0,
    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    memoryLimit BIGINT DEFAULT 0,
+    altDomain VARCHAR(256),
+    xFrameOptions VARCHAR(512),

-    lastBackupId VARCHAR(128),
-    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupId VARCHAR(128), // tracks last valid backup, can be removed

-    oldConfigJson TEXT, // used to pass old config for apptask
+    oldConfigJson TEXT, // used to pass old config for apptask, can be removed when we use a queue

    PRIMARY KEY(id));

@@ -71,7 +86,7 @@ CREATE TABLE IF NOT EXISTS authcodes(
    authCode VARCHAR(128) NOT NULL UNIQUE,
    userId VARCHAR(128) NOT NULL,
    clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
    PRIMARY KEY(authCode));

 CREATE TABLE IF NOT EXISTS settings(
@@ -85,3 +100,32 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
    value VARCHAR(512) NOT NULL,
    FOREIGN KEY(appId) REFERENCES apps(id));

+CREATE TABLE IF NOT EXISTS backups(
+    filename VARCHAR(128) NOT NULL,
+    creationTime TIMESTAMP,
+    version VARCHAR(128) NOT NULL, /* app version or box version */
+    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
+    dependsOn VARCHAR(4096), /* comma separate list of objects this backup depends on */
+    state VARCHAR(16) NOT NULL,
+
+    PRIMARY KEY (filename));
+
+CREATE TABLE IF NOT EXISTS eventlog(
+    id VARCHAR(128) NOT NULL,
+    action VARCHAR(128) NOT NULL,
+    source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+
+    PRIMARY KEY (id));
+
+/* Future fields:
+   * accessRestriction - to determine who can access it. So this has foreign keys
+   * quota - per mailbox quota
+*/
+CREATE TABLE IF NOT EXISTS mailboxes(
+    name VARCHAR(128) NOT NULL,
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
+    creationTime TIMESTAMP,
+
+    PRIMARY KEY (id));
@@ -16,10 +16,9 @@
    "async": "^1.2.1",
    "aws-sdk": "^2.1.46",
    "body-parser": "^1.13.1",
-    "bytes": "^2.1.0",
-    "cloudron-manifestformat": "^2.0.0",
+    "cloudron-manifestformat": "^2.4.0",
    "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
+    "connect-lastmile": "^0.1.0",
    "connect-timeout": "^1.5.0",
    "cookie-parser": "^1.3.5",
    "cookie-session": "^1.1.0",
@@ -27,54 +26,58 @@
    "csurf": "^1.6.6",
    "db-migrate": "^0.9.2",
    "debug": "^2.2.0",
-    "dockerode": "^2.2.2",
+    "dockerode": "^2.2.10",
    "ejs": "^2.2.4",
-    "ejs-cli": "^1.0.1",
+    "ejs-cli": "^1.2.0",
    "express": "^4.12.4",
    "express-session": "^1.11.3",
    "hat": "0.0.3",
+    "ini": "^1.3.4",
    "json": "^9.0.3",
    "ldapjs": "^0.7.1",
-    "memorystream": "^0.3.0",
    "mime": "^1.3.4",
-    "morgan": "^1.6.0",
+    "morgan": "^1.7.0",
    "multiparty": "^4.1.2",
    "mysql": "^2.7.0",
    "native-dns": "^0.7.0",
+    "node-df": "^0.1.1",
    "node-uuid": "^1.4.3",
    "nodemailer": "^1.3.0",
    "nodemailer-smtp-transport": "^1.0.3",
    "oauth2orize": "^1.0.1",
    "once": "^1.3.2",
+    "parse-links": "^0.1.0",
    "passport": "^0.2.2",
    "passport-http": "^0.2.2",
    "passport-http-bearer": "^1.0.1",
    "passport-local": "^1.0.0",
    "passport-oauth2-client-password": "^0.1.2",
-    "password-generator": "^1.0.0",
+    "password-generator": "^2.0.2",
    "proxy-middleware": "^0.13.0",
-    "safetydance": "0.0.19",
+    "safetydance": "^0.1.1",
    "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
    "split": "^1.0.0",
-    "superagent": "~0.21.0",
+    "superagent": "^1.8.3",
    "supererror": "^0.7.1",
    "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "tldjs": "^1.6.2",
    "underscore": "^1.7.0",
+    "ursa": "^0.9.3",
    "valid-url": "^1.0.9",
-    "validator": "^3.30.0",
-    "x509": "^0.2.2"
+    "validator": "^4.9.0",
+    "x509": "^0.2.4"
  },
  "devDependencies": {
    "apidoc": "*",
    "bootstrap-sass": "^3.3.3",
+    "deep-extend": "^0.4.1",
    "del": "^1.1.1",
    "expect.js": "*",
    "gulp": "^3.8.11",
    "gulp-autoprefixer": "^2.3.0",
    "gulp-concat": "^2.4.3",
+    "gulp-cssnano": "^2.1.0",
    "gulp-ejs": "^1.0.0",
-    "gulp-minify-css": "^1.1.3",
    "gulp-sass": "^2.0.1",
    "gulp-serve": "^1.0.0",
    "gulp-sourcemaps": "^1.5.2",
@@ -83,9 +86,8 @@
    "istanbul": "*",
    "js2xmlparser": "^1.0.0",
    "mocha": "*",
-    "nock": "^2.6.0",
+    "nock": "^3.4.0",
    "node-sass": "^3.0.0-alpha.0",
-    "redis": "^0.12.1",
    "request": "^2.65.0",
    "sinon": "^1.12.2",
    "yargs": "^3.15.0"
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+set -eu
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revision:,output:,publish,no-upload" -n "$0" -- "$@")
+eval set -- "${args}"
+
+readonly RELEASE_TOOL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../release" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+delete_bundle="yes"
+commitish="HEAD"
+publish="no"
+upload="yes"
+bundle_file=""
+
+while true; do
+    case "$1" in
+    --revision) commitish="$2"; shift 2;;
+    --output) bundle_file="$2"; delete_bundle="no"; shift 2;;
+    --no-upload) upload="no"; shift;;
+    --publish) publish="yes"; shift;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+if [[ "${upload}" == "no" && "${publish}" == "yes" ]]; then
+    echo "Cannot publish without uploading"
+    exit 1
+fi
+
+readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint?
+
+assertNotEmpty AWS_DEV_ACCESS_KEY
+assertNotEmpty AWS_DEV_SECRET_KEY
+
+if ! $(cd "${SOURCE_DIR}" && git diff --exit-code >/dev/null); then
+    echo "You have local changes, stash or commit them to proceed"
+    exit 1
+fi
+
+if [[ "$(node --version)" != "v4.1.1" ]]; then
+    echo "This script requires node 4.1.1"
+    exit 1
+fi
+
+version=$(cd "${SOURCE_DIR}" && git rev-parse "${commitish}")
+bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
+[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${version}.tar.gz"
+
+chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
+echo "Checking out code [${version}] into ${bundle_dir}"
+(cd "${SOURCE_DIR}" && git archive --format=tar ${version} | (cd "${bundle_dir}" && tar xf -))
+
+if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then
+    echo "Reusing dev modules from cache"
+    cp -r "${TMPDIR}/boxtarball.cache/node_modules-all/." "${bundle_dir}/node_modules"
+else
+    echo "Installing modules with dev dependencies"
+    (cd "${bundle_dir}" && npm install)
+
+    echo "Caching dev dependencies"
+    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-all"
+    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-all/"
+    cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all"
+fi
+
+echo "Building webadmin assets"
+(cd "${bundle_dir}" && gulp)
+
+echo "Remove intermediate files required at build-time only"
+rm -rf "${bundle_dir}/node_modules/"
+rm -rf "${bundle_dir}/webadmin/src"
+rm -rf "${bundle_dir}/gulpfile.js"
+
+if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then
+    echo "Reusing prod modules from cache"
+    cp -r "${TMPDIR}/boxtarball.cache/node_modules-prod/." "${bundle_dir}/node_modules"
+else
+    echo "Installing modules for production"
+    (cd "${bundle_dir}" && npm install --production --no-optional)
+
+    echo "Caching prod dependencies"
+    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-prod"
+    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-prod/"
+    cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod"
+fi
+
+echo "Create final tarball"
+(cd "${bundle_dir}" && tar czf "${bundle_file}" .)
+echo "Cleaning up ${bundle_dir}"
+rm -rf "${bundle_dir}"
+
+if [[ "${upload}" == "yes" ]]; then
+    echo "Uploading bundle to S3"
+    # That special header is needed to allow access with singed urls created with different aws credentials than the ones the file got uploaded
+    s3cmd --multipart-chunk-size-mb=5 --ssl --acl-public --access_key="${AWS_DEV_ACCESS_KEY}" --secret_key="${AWS_DEV_SECRET_KEY}" --no-mime-magic put "${bundle_file}" "s3://dev-cloudron-releases/box-${version}.tar.gz"
+
+    versions_file_url="https://dev-cloudron-releases.s3.amazonaws.com/box-${version}.tar.gz"
+    echo "The URL for the versions file is: ${versions_file_url}"
+
+    if [[ "${publish}" == "yes" ]]; then
+        echo "Publishing to dev"
+        ${RELEASE_TOOL_DIR}/release create --env dev --code "${versions_file_url}"
+    fi
+fi
+
+if [[ "${delete_bundle}" == "no" ]]; then
+    echo "Tarball preserved at ${bundle_file}"
+else
+    rm "${bundle_file}"
+fi
+
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=21
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.8.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.8.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.9.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite
@@ -10,7 +10,9 @@ arg_fqdn=""
 arg_is_custom_domain="false"
 arg_restore_key=""
 arg_restore_url=""
-arg_retire="false"
+arg_retire_reason=""
+arg_retire_info=""
+arg_tls_config=""
 arg_tls_cert=""
 arg_tls_key=""
 arg_token=""
@@ -18,24 +20,45 @@ arg_version=""
 arg_web_server_origin=""
 arg_backup_config=""
 arg_dns_config=""
+arg_update_config=""
+arg_provider=""
+arg_app_bundle=""

-args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
+args=$(getopt -o "" -l "data:,retire-reason:,retire-info:" -n "$0" -- "$@")
 eval set -- "${args}"

 while true; do
    case "$1" in
-    --retire)
-        arg_retire="true"
-        shift
+    --retire-reason)
+        arg_retire_reason="$2"
+        shift 2
+        ;;
+    --retire-info)
+        arg_retire_info="$2"
+        shift 2
        ;;
    --data)
-        # only read mandatory non-empty parameters here
-        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_token arg_is_custom_domain arg_box_versions_url arg_version <<EOF
-        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn token isCustomDomain boxVersionsUrl version | tr '\n' ' ')
-EOF
+        # these params must be valid in all cases
+        arg_fqdn=$(echo "$2" | $json fqdn)
+        arg_is_custom_domain=$(echo "$2" | $json isCustomDomain)
+
+        # only update/restore have this valid (but not migrate)
+        arg_api_server_origin=$(echo "$2" | $json apiServerOrigin)
+        arg_web_server_origin=$(echo "$2" | $json webServerOrigin)
+        arg_box_versions_url=$(echo "$2" | $json boxVersionsUrl)
+        arg_version=$(echo "$2" | $json version)
+
        # read possibly empty parameters here
+        arg_app_bundle=$(echo "$2" | $json appBundle)
+        [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"
+
        arg_tls_cert=$(echo "$2" | $json tlsCert)
        arg_tls_key=$(echo "$2" | $json tlsKey)
+        arg_token=$(echo "$2" | $json token)
+        arg_provider=$(echo "$2" | $json provider)
+
+        arg_tls_config=$(echo "$2" | $json tlsConfig)
+        [[ "${arg_tls_config}" == "null" ]] && arg_tls_config=""

        arg_restore_url=$(echo "$2" | $json restore.url)
        [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
@@ -49,6 +72,9 @@ EOF
        arg_dns_config=$(echo "$2" | $json dnsConfig)
        [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""

+        arg_update_config=$(echo "$2" | $json updateConfig)
+        [[ "${arg_update_config}" == "null" ]] && arg_update_config=""
+
        shift 2
        ;;
    --) break;;
@@ -66,5 +92,7 @@ echo "restore url: ${arg_restore_url}"
 echo "tls cert: ${arg_tls_cert}"
 echo "tls key: ${arg_tls_key}"
 echo "token: ${arg_token}"
+echo "tlsConfig: ${arg_tls_config}"
 echo "version: ${arg_version}"
 echo "web server: ${arg_web_server_origin}"
+echo "provider: ${arg_provider}"
@@ -20,7 +20,7 @@ systemctl daemon-reload
 systemctl enable cloudron.target

 ########## sudoers
-rm -f /etc/sudoers.d/*
+rm -f /etc/sudoers.d/yellowtent
 cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent

 ########## collectd
@@ -4,4 +4,4 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connection=50
+max_connections=50
@@ -1,3 +1,6 @@
+# sudo logging breaks journalctl output with very long urls (systemd bug)
+Defaults !syslog
+
 Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh

@@ -22,8 +25,9 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh

-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
+
+Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
+
@@ -2,6 +2,8 @@
 Description=Cloudron Admin
 OnFailure=crashnotifier@%n.service
 StopWhenUnneeded=true
+; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
+BindsTo=systemd-journald.service

 [Service]
 Type=idle
@@ -9,9 +11,12 @@ WorkingDirectory=/home/yellowtent/box
 Restart=always
 ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
-KillMode=process
+; kill apptask processes as well
+KillMode=control-group
 User=yellowtent
 Group=yellowtent
 MemoryLimit=200M
 TimeoutStopSec=5s
+StartLimitInterval=1
+StartLimitBurst=60

@@ -1,5 +1,5 @@
 [Unit]
-Description=Cloudron Smart Cloud
+Description=Cloudron Smartserver
 Documentation=https://cloudron.io/documentation.html
 StopWhenUnneeded=true
 Requires=box.service
@@ -7,7 +7,7 @@ StopWhenUnneeded=false
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
-ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+ExecStart="/home/yellowtent/box/crashnotifierservice.js" %I
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 KillMode=process
 User=yellowtent
@@ -9,8 +9,6 @@ readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
 readonly ADMIN_LOCATION="my" # keep this in sync with constants.js

-source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
-
 echo "Setting up nginx update page"

 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
@@ -24,17 +22,24 @@ rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"

 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+readonly current_infra=$(node -e "console.log(require('${script_dir}/../src/infra_version.js').version);")
+existing_infra="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && existing_infra=$(node -e "console.log(JSON.parse(require('fs').readFileSync('${DATA_DIR}/INFRA_VERSION', 'utf8')).version);")
+if [[ "${arg_retire_reason}" != "" || "${existing_infra}" != "${current_infra}" ]]; then
+    echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire_reason} existing: ${existing_infra} current: ${current_infra}"
    rm -f ${DATA_DIR}/nginx/applications/*
    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
+    echo "Show progress bar only on admin domain for normal update"
    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi

-echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+if [[ "${arg_retire_reason}" == "migrate" ]]; then
+    echo "{ \"migrate\": { \"percent\": \"10\", \"message\": \"Migrating cloudron. This could take up to 15 minutes.\", \"info\": ${arg_retire_info} }, \"backup\": null, \"apiServerOrigin\": \"${arg_api_server_origin}\" }" > "${SETUP_WEBSITE_DIR}/progress.json"
+else
+    echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+fi

 nginx -s reload
@@ -34,12 +34,21 @@ set_progress() {
 set_progress "1" "Create container"
 $script_dir/container.sh

+set_progress "5" "Adjust system settings"
+hostnamectl set-hostname "${arg_fqdn}"
+
+# ec2 instances use lots of cpu for swapping, which can be significantly reduced adjusting the swappiness
+if [[ "${arg_provider}" == 'ec2' ]]; then
+    sysctl vm.swappiness=0
+fi
+
 set_progress "10" "Ensuring directories"
 # keep these in sync with paths.js
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
 mkdir -p "${DATA_DIR}/box/appicons"
 mkdir -p "${DATA_DIR}/box/certs"
-mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/box/mail/dkim/${arg_fqdn}"
+mkdir -p "${DATA_DIR}/box/acme" # acme keys
 mkdir -p "${DATA_DIR}/graphite"

 mkdir -p "${DATA_DIR}/mysql"
@@ -48,6 +57,7 @@ mkdir -p "${DATA_DIR}/mongodb"
 mkdir -p "${DATA_DIR}/snapshots"
 mkdir -p "${DATA_DIR}/addons"
 mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${DATA_DIR}/acme" # acme challenges

 # bookkeep the version as part of data
 echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
@@ -90,30 +100,22 @@ EOF

 set_progress "28" "Setup collectd"
 cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
-# collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
-mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
-vda1_id=$(blkid -s UUID -o value /dev/vda1)
-ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
 service collectd restart

 set_progress "30" "Setup nginx"
-# setup naked domain to use admin by default. app restoration will overwrite this config
 mkdir -p "${DATA_DIR}/nginx/applications"
+cp "${script_dir}/start/nginx/nginx.conf" "${DATA_DIR}/nginx/nginx.conf"
 cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"

-# generate the main nginx config file
-${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs" \
-    -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
-
 # generate these for update code paths as well to overwrite splash
-admin_cert_file="cert/host.cert"
-admin_key_file="cert/host.key"
-if [[ -f "${DATA_DIR}/box/certs/admin.cert" && -f "${DATA_DIR}/box/certs/admin.key" ]]; then
-    admin_cert_file="${DATA_DIR}/box/certs/admin.cert"
-    admin_key_file="${DATA_DIR}/box/certs/admin.key"
+admin_cert_file="${DATA_DIR}/nginx/cert/host.cert"
+admin_key_file="${DATA_DIR}/nginx/cert/host.key"
+if [[ -f "${DATA_DIR}/box/certs/${admin_fqdn}.cert" && -f "${DATA_DIR}/box/certs/${admin_fqdn}.key" ]]; then
+    admin_cert_file="${DATA_DIR}/box/certs/${admin_fqdn}.cert"
+    admin_key_file="${DATA_DIR}/box/certs/${admin_fqdn}.key"
 fi
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\", \"xFrameOptions\": \"SAMEORIGIN\" }" > "${DATA_DIR}/nginx/applications/admin.conf"

 mkdir -p "${DATA_DIR}/nginx/cert"
 if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
@@ -125,12 +127,10 @@ else
 fi

 set_progress "33" "Changing ownership"
-chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"

-set_progress "40" "Setting up infra"
-${script_dir}/start/setup_infra.sh "${arg_fqdn}"
-
 set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
@@ -144,14 +144,15 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
    "fqdn": "${arg_fqdn}",
    "isCustomDomain": ${arg_is_custom_domain},
    "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "admin@${arg_fqdn}",
+    "provider": "${arg_provider}",
    "database": {
        "hostname": "localhost",
        "username": "root",
        "password": "${mysql_root_password}",
        "port": 3306,
        "name": "box"
-    }
+    },
+    "appBundle": ${arg_app_bundle}
 }
 CONF_END

@@ -179,18 +180,36 @@ if [[ ! -z "${arg_dns_config}" ]]; then
        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
 fi

-# Add webadmin oauth client
+# Add Update Configuration
+if [[ ! -z "${arg_update_config}" ]]; then
+    echo "Add Update Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"update_config\", '$arg_update_config')" box
+fi
+
+# Add TLS Configuration
+if [[ ! -z "${arg_tls_config}" ]]; then
+    echo "Add TLS Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
+fi
+
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add webadmin api cient"
+readonly ADMIN_SCOPES="cloudron,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"Settings\", \"built-in\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box

-echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add SDK api client"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-sdk\", \"SDK\", \"built-in\", \"secret-sdk\", \"${admin_origin}\", \"*,roleSdk\")" box
+
+echo "Add cli api client"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-cli\", \"Cloudron Tool\", \"built-in\", \"secret-cli\", \"${admin_origin}\", \"*,roleSdk\")" box

 set_progress "80" "Starting Cloudron"
 systemctl start cloudron.target
@@ -133,10 +133,10 @@ LoadPlugin nginx
 #   Globals true
 #</LoadPlugin>
 #LoadPlugin pinba
-LoadPlugin ping
+#LoadPlugin ping
 #LoadPlugin postgresql
 #LoadPlugin powerdns
-LoadPlugin processes
+#LoadPlugin processes
 #LoadPlugin protocols
 #<LoadPlugin python>
 #   Globals true
@@ -161,7 +161,7 @@ LoadPlugin tail
 #LoadPlugin users
 #LoadPlugin uuid
 #LoadPlugin varnish
-LoadPlugin vmem
+#LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
 LoadPlugin write_graphite
@@ -193,11 +193,11 @@ LoadPlugin write_graphite
 </Plugin>

 <Plugin df>
-   FSType "tmpfs"
-   MountPoint "/dev"
+   FSType "ext4"
+   FSType "btrfs"

   ReportByDevice true
-   IgnoreSelected true
+   IgnoreSelected false

   ValuesAbsolute true
   ValuesPercentage true
@@ -212,17 +212,6 @@ LoadPlugin write_graphite
   URL "http://127.0.0.1/nginx_status"
 </Plugin>

-<Plugin ping>
-   Host "google.com"
-   Interval 1.0
-   Timeout 0.9
-   TTL 255
-</Plugin>
-
-<Plugin processes>
-   ProcessMatch "app" "node box.js"
-</Plugin>
-
 <Plugin swap>
   ReportByDevice false
   ReportBytes true
@@ -255,10 +244,6 @@ LoadPlugin write_graphite
   </File>
 </Plugin>

-<Plugin vmem>
-   Verbose false
-</Plugin>
-
 <Plugin write_graphite>
   <Node "graphing">
       Host "localhost"
@@ -18,11 +18,15 @@ server {
    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
    # https://cipherli.st/
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
    ssl_prefer_server_ciphers on;
    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";

+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/X-Frame-Options
+    add_header X-Frame-Options "<%= xFrameOptions %>";
+
    proxy_http_version 1.1;
    proxy_intercept_errors on;
    proxy_read_timeout       3500;
@@ -58,12 +62,18 @@ server {
            client_max_body_size 1m;
        }

-        # graphite paths
-        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
-            proxy_pass   http://127.0.0.1:8000;
-            client_max_body_size 1m;
+        # the read timeout is between successive reads and not the whole connection
+        location ~ ^/api/v1/apps/.*/exec$ {
+            proxy_pass   http://127.0.0.1:3000;
+            proxy_read_timeout 30m;
        }

+        # graphite paths
+        # location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+        #     proxy_pass   http://127.0.0.1:8000;
+        #     client_max_body_size 1m;
+        # }
+
        location / {
            root   <%= sourceDir %>/webadmin/dist;
            index  index.html index.htm;
@@ -24,7 +24,14 @@ http {

    sendfile        on;

-    keepalive_timeout  65;
+    # timeout for client to finish sending headers
+    client_header_timeout 30s;
+
+    # timeout for reading client request body (successive read timeout and not whole body!)
+    client_body_timeout 60s;
+
+    # keep-alive connections timeout in 65s. this is because many browsers timeout in 60 seconds
+    keepalive_timeout  65s;

    # HTTP server
    server {
@@ -38,14 +45,21 @@ http {
            deny all;
        }

+        # acme challenges
+        location /.well-known/acme-challenge/ {
+            default_type text/plain;
+            alias /home/yellowtent/data/acme/;
+        }
+
        location / {
            # redirect everything to HTTPS
            return 301 https://$host$request_uri;
        }
    }

-    # We have to enable https for nginx to read in the vhost in http request
-    # and send a 404. This is a side-effect of using wildcard DNS
+    # This server handles the naked domain for custom domains.
+    # It can also be used for wildcard subdomain 404. This feature is not used by the Cloudron itself
+    # because box always sets up DNS records for app subdomains.
    server {
        listen 443 default_server;
        ssl on;
@@ -55,11 +69,21 @@ http {
        error_page 404 = @fallback;
        location @fallback {
            internal;
-            root <%= sourceDir %>/webadmin/dist;
+            root /home/yellowtent/box/webadmin/dist;
            rewrite ^/$ /nakeddomain.html break;
        }

-        return 404;
+        location / {
+            internal;
+            root /home/yellowtent/box/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
+        # required for /api/v1/cloudron/avatar
+        location /api/ {
+            proxy_pass http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
    }

    include applications/*.conf;
@@ -1,129 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-arg_fqdn="$1"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"
@@ -7,7 +7,6 @@ exports = module.exports = {
    restoreAddons: restoreAddons,

    getEnvironment: getEnvironment,
-    getLinksSync: getLinksSync,
    getBindsSync: getBindsSync,
    getContainerNamesSync: getContainerNamesSync,

@@ -19,30 +18,34 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
    assert = require('assert'),
    async = require('async'),
-    child_process = require('child_process'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
-    DatabaseError = require('./databaseerror.js'),
+    ClientsError = clients.ClientsError,
    debug = require('debug')('box:addons'),
-    docker = require('./docker.js').connection,
+    docker = require('./docker.js'),
+    dockerConnection = docker.connection,
    fs = require('fs'),
    generatePassword = require('password-generator'),
    hat = require('hat'),
-    MemoryStream = require('memorystream'),
+    infra = require('./infra_version.js'),
    once = require('once'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    shell = require('./shell.js'),
-    spawn = child_process.spawn,
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');

 var NOOP = function (app, options, callback) { return callback(); };

 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    email: {
+        setup: setupEmail,
+        teardown: teardownEmail,
+        backup: NOOP,
+        restore: setupEmail
+    },
    ldap: {
        setup: setupLdap,
        teardown: teardownLdap,
@@ -79,6 +82,12 @@ var KNOWN_ADDONS = {
        backup: backupPostgreSql,
        restore: restorePostgreSql
    },
+    recvmail: {
+        setup: setupRecvMail,
+        teardown: teardownRecvMail,
+        backup: NOOP,
+        restore: setupRecvMail
+    },
    redis: {
        setup: setupRedis,
        teardown: teardownRedis,
@@ -199,28 +208,6 @@ function getEnvironment(app, callback) {
    appdb.getAddonConfigByAppId(app.id, callback);
 }

-function getLinksSync(app, addons) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addons || typeof addons === 'object');
-
-    var links = [ ];
-
-    if (!addons) return links;
-
-    for (var addon in addons) {
-        switch (addon) {
-        case 'mysql': links.push('mysql:mysql'); break;
-        case 'postgresql': links.push('postgresql:postgresql'); break;
-        case 'sendmail': links.push('mail:mail'); break;
-        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
-        case 'mongodb': links.push('mongodb:mongodb'); break;
-        default: break;
-        }
-    }
-
-    return links;
-}
-
 function getBindsSync(app, addons) {
    assert.strictEqual(typeof app, 'object');
    assert(!addons || typeof addons === 'object');
@@ -267,22 +254,18 @@ function setupOauth(app, options, callback) {
    assert.strictEqual(typeof callback, 'function');

    var appId = app.id;
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
    var redirectURI = 'https://' + config.appFqdn(app.location);
    var scope = 'profile';

-    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
+    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);

-    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
            if (error) return callback(error);

            var env = [
-                'OAUTH_CLIENT_ID=' + id,
-                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_CLIENT_ID=' + result.id,
+                'OAUTH_CLIENT_SECRET=' + result.clientSecret,
                'OAUTH_ORIGIN=' + config.adminOrigin()
            ];

@@ -300,8 +283,8 @@ function teardownOauth(app, options, callback) {

    debugApp(app, 'teardownOauth');

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);

        appdb.unsetAddonConfig(app.id, 'oauth', callback);
    });
@@ -313,23 +296,20 @@ function setupSimpleAuth(app, options, callback) {
    assert.strictEqual(typeof callback, 'function');

    var appId = app.id;
-    var id = 'cid-' + uuid.v4();
    var scope = 'profile';

-    debugApp(app, 'setupSimpleAuth: id:%s', id);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+        clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) {
            if (error) return callback(error);

            var env = [
-                'SIMPLE_AUTH_SERVER=172.17.0.1',
+                'SIMPLE_AUTH_SERVER=172.18.0.1',
                'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
-                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_CLIENT_ID=' + id
+                'SIMPLE_AUTH_URL=http://172.18.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.18.0.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + result.id
            ];

            debugApp(app, 'Setting simple auth addon config to %j', env);
@@ -346,26 +326,57 @@ function teardownSimpleAuth(app, options, callback) {

    debugApp(app, 'teardownSimpleAuth');

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);

        appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
    });
 }

+function setupEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2525',
+        'MAIL_IMAP_SERVER=mail',
+        'MAIL_IMAP_PORT=9993',
+        'MAIL_SIEVE_SERVER=mail',
+        'MAIL_SIEVE_PORT=4190',
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up Email');
+
+    appdb.setAddonConfig(app.id, 'email', env, callback);
+}
+
+function teardownEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down Email');
+
+    appdb.unsetAddonConfig(app.id, 'email', callback);
+}
+
 function setupLdap(app, options, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

    var env = [
-        'LDAP_SERVER=172.17.0.1',
+        'LDAP_SERVER=172.18.0.1',
        'LDAP_PORT=' + config.get('ldapPort'),
-        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.18.0.1:' + config.get('ldapPort'),
        'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
        'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
-        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
+        'LDAP_BIND_PASSWORD=' + hat(8 * 128) // this is ignored
    ];

    debugApp(app, 'Setting up LDAP');
@@ -388,16 +399,17 @@ function setupSendMail(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var env = [
-        'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + (app.location || app.id) + '-app', // use app.id for bare domains
-        'MAIL_DOMAIN=' + config.fqdn()
-    ];
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';

-    debugApp(app, 'Setting up sendmail');
+    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];

-    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting sendmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    });
 }

 function teardownSendMail(app, options, callback) {
@@ -405,9 +417,55 @@ function teardownSendMail(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    debugApp(app, 'Tearing down sendmail');
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';

-    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
+
+    debugApp(app, 'Tearing down sendmail : %j', cmd);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    });
+}
+
+function setupRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up recvmail');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
+
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting recvmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'recvmail', env, callback);
+    });
+}
+
+function teardownRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
+
+    debugApp(app, 'Tearing down recvmail: %j', cmd);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
+    });
 }

 function setupMySql(app, options, callback) {
@@ -417,31 +475,14 @@ function setupMySql(app, options, callback) {

    debugApp(app, 'Setting up mysql');

-    var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'add', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { bufferStdout: true }, function (error, stdout) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mysql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mysql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mysql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mysql', env, callback);
    });
 }

@@ -450,24 +491,14 @@ function teardownMySql(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];

    debugApp(app, 'Tearing down mysql');

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { }, function (error) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mysql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mysql', callback);
    });
 }

@@ -479,15 +510,9 @@ function backupMySql(app, options, callback) {
    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
    output.on('error', callback);

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ];

-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mysql', cmd, { stdout: output }, callback);
 }

 function restoreMySql(app, options, callback) {
@@ -501,17 +526,8 @@ function restoreMySql(app, options, callback) {
        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
        input.on('error', callback);

-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMySql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ];
+        docker.execContainer('mysql', cmd, { stdin: input }, callback);
    });
 }

@@ -522,31 +538,14 @@ function setupPostgreSql(app, options, callback) {

    debugApp(app, 'Setting up postgresql');

-    var container = docker.getContainer('postgresql');
    var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { bufferStdout: true }, function (error, stdout) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting postgresql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'postgresql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting postgresql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'postgresql', env, callback);
    });
 }

@@ -555,24 +554,14 @@ function teardownPostgreSql(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = docker.getContainer('postgresql');
    var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];

    debugApp(app, 'Tearing down postgresql');

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { }, function (error) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'postgresql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'postgresql', callback);
    });
 }

@@ -584,19 +573,13 @@ function backupPostgreSql(app, options, callback) {
    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
    output.on('error', callback);

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/postgresql/service.sh', 'backup', app.id ];

-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('postgresql', cmd, { stdout: output }, callback);
 }

 function restorePostgreSql(app, options, callback) {
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    callback = once(callback);

    setupPostgreSql(app, options, function (error) {
        if (error) return callback(error);
@@ -606,17 +589,9 @@ function restorePostgreSql(app, options, callback) {
        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
        input.on('error', callback);

-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
-        });
+        var cmd = [ '/addons/postgresql/service.sh', 'restore', app.id ];

-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        docker.execContainer('postgresql', cmd, { stdin: input }, callback);
    });
 }

@@ -627,31 +602,14 @@ function setupMongoDb(app, options, callback) {

    debugApp(app, 'Setting up mongodb');

-    var container = docker.getContainer('mongodb');
    var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { bufferStdout: true }, function (error, stdout) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mongodb addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mongodb addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mongodb', env, callback);
    });
 }

@@ -660,24 +618,14 @@ function teardownMongoDb(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = docker.getContainer('mongodb');
    var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];

    debugApp(app, 'Tearing down mongodb');

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { }, function (error) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mongodb', callback);
    });
 }

@@ -689,15 +637,9 @@ function backupMongoDb(app, options, callback) {
    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
    output.on('error', callback);

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mongodb/service.sh', 'backup', app.id ];

-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mongodb', cmd, { stdout: output }, callback);
 }

 function restoreMongoDb(app, options, callback) {
@@ -711,60 +653,18 @@ function restoreMongoDb(app, options, callback) {
        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
        input.on('error', callback);

-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mongodb/service.sh', 'restore', app.id ];
+        docker.execContainer('mongodb', cmd, { stdin: input }, callback);
    });
 }

-
-function forwardRedisPort(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    docker.getContainer('redis-' + appId).inspect(function (error, data) {
-        if (error) return callback(new Error('Unable to inspect container:' + error));
-
-        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
-        if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
-
-        return callback(null);
-    });
-}
-
-function stopAndRemoveRedis(container, callback) {
-    function ignoreError(func) {
-        return function (callback) {
-            func(function (error) {
-                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
-                callback();
-            });
-        };
-    }
-
-    // stopping redis with SIGTERM makes it commit the database to disk
-    async.series([
-        ignoreError(container.stop.bind(container, { t: 10 })),
-        ignoreError(container.wait.bind(container)),
-        ignoreError(container.remove.bind(container, { force: true, v: true }))
-    ], callback);
-}
-
 // Ensures that app's addon redis container is running. Can be called when named container already exists/running
 function setupRedis(app, options, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var redisPassword = generatePassword(64, false /* memorable */);
+    var redisPassword = generatePassword(64, false /* memorable */, /[\w\d_]/); // ensure no / in password for being sed friendly (and be uri friendly)
    var redisVarsFile = path.join(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
    var redisDataDir = path.join(paths.DATA_DIR, app.id + '/redis');

@@ -774,57 +674,32 @@ function setupRedis(app, options, callback) {

    if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error));

-    var createOptions = {
-        name: 'redis-' + app.id,
-        Hostname: 'redis-' + app.location,
-        Tty: true,
-        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
-        Cmd: null,
-        Volumes: {
-            '/tmp': {},
-            '/run': {}
-        },
-        VolumesFrom: [],
-        HostConfig: {
-            Binds: [
-                redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
-                redisDataDir + ':/var/lib/redis:rw'
-            ],
-            Memory: 1024 * 1024 * 75, // 100mb
-            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
-            PortBindings: {
-                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
-            },
-            ReadonlyRootfs: true,
-            RestartPolicy: {
-                'Name': 'always',
-                'MaximumRetryCount': 0
-            }
-        }
-    };
+    const tag = infra.images.redis.tag, redisName = 'redis-' + app.id;
+    const cmd = `docker run --restart=always -d --name=${redisName} \
+                --net cloudron \
+                --net-alias ${redisName} \
+                -m 100m \
+                --memory-swap 150m \
+                -v ${redisVarsFile}:/etc/redis/redis_vars.sh:ro \
+                -v ${redisDataDir}:/var/lib/redis:rw \
+                --read-only -v /tmp -v /run ${tag}`;

    var env = [
        'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id,
        'REDIS_PASSWORD=' + redisPassword,
-        'REDIS_HOST=redis-' + app.id,
+        'REDIS_HOST=' + redisName,
        'REDIS_PORT=6379'
    ];

-    var redisContainer = docker.getContainer(createOptions.name);
-    stopAndRemoveRedis(redisContainer, function () {
-        docker.createContainer(createOptions, function (error) {
-            if (error && error.statusCode !== 409) return callback(error); // if not already created
-
-            redisContainer.start(function (error) {
-                if (error && error.statusCode !== 304) return callback(error); // if not already running
-
-                appdb.setAddonConfig(app.id, 'redis', env, function (error) {
-                    if (error) return callback(error);
-
-                    forwardRedisPort(app.id, callback);
-                });
-            });
-        });
+    async.series([
+        // stop so that redis can flush itself with SIGTERM
+        shell.execSync.bind(null, 'stopRedis', `docker stop --time=10 ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'stopRedis', `docker rm --volumes ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'startRedis', cmd),
+        appdb.setAddonConfig.bind(null, app.id, 'redis', env)
+    ], function (error) {
+        if (error) debug('Error setting up redis: ', error);
+        callback(error);
    });
 }

@@ -833,7 +708,7 @@ function teardownRedis(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-   var container = docker.getContainer('redis-' + app.id);
+   var container = dockerConnection.getContainer('redis-' + app.id);

   var removeOptions = {
       force: true, // kill container if it's running
@@ -856,15 +731,7 @@ function teardownRedis(app, options, callback) {
 function backupRedis(app, options, callback) {
    debugApp(app, 'Backing up redis');

-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    var cmd = [ '/addons/redis/service.sh', 'backup' ]; // the redis dir is volume mounted

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
-    });
-
-    cp.stdout.pipe(process.stdout);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('redis-' + app.id, cmd, { }, callback);
 }
@@ -4,7 +4,6 @@

 exports = module.exports = {
    get: get,
-    getBySubdomain: getBySubdomain,
    getByHttpPort: getByHttpPort,
    getByContainerId: getByContainerId,
    add: add,
@@ -27,6 +26,7 @@ exports = module.exports = {

    // installation codes (keep in sync in UI)
    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CLONE: 'pending_clone', // clone
    ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
@@ -59,7 +59,7 @@ var assert = require('assert'),

 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain', 'apps.xFrameOptions' ].join(',');

 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');

@@ -70,10 +70,6 @@ function postProcess(result) {
    result.manifest = safe.JSON.parse(result.manifestJson);
    delete result.manifestJson;

-    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
-    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
-    delete result.lastBackupConfigJson;
-
    assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
    result.oldConfig = safe.JSON.parse(result.oldConfigJson);
    delete result.oldConfigJson;
@@ -92,12 +88,13 @@ function postProcess(result) {
        result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
    }

-    result.oauthProxy = !!result.oauthProxy;
-
    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
    delete result.accessRestrictionJson;
+
+    // TODO remove later once all apps have this attribute
+    result.xFrameOptions = result.xFrameOptions || 'SAMEORIGIN';
 }

 function get(id, callback) {
@@ -116,22 +113,6 @@ function get(id, callback) {
    });
 }

-function getBySubdomain(subdomain, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
 function getByHttpPort(httpPort, callback) {
    assert.strictEqual(typeof httpPort, 'number');
    assert.strictEqual(typeof callback, 'function');
@@ -179,26 +160,32 @@ function getAll(callback) {
    });
 }

-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
+function add(id, appStoreId, manifest, location, portBindings, data, callback) {
    assert.strictEqual(typeof id, 'string');
    assert.strictEqual(typeof appStoreId, 'string');
    assert(manifest && typeof manifest === 'object');
    assert.strictEqual(typeof manifest.version, 'string');
    assert.strictEqual(typeof location, 'string');
    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert(data && typeof data === 'object');
    assert.strictEqual(typeof callback, 'function');

    portBindings = portBindings || { };

    var manifestJson = JSON.stringify(manifest);
+
+    var accessRestriction = data.accessRestriction || null;
    var accessRestrictionJson = JSON.stringify(accessRestriction);
+    var memoryLimit = data.memoryLimit || 0;
+    var altDomain = data.altDomain || null;
+    var xFrameOptions = data.xFrameOptions || '';
+    var installationState = data.installationState || exports.ISTATE_PENDING_INSTALL;
+    var lastBackupId = data.lastBackupId || null; // used when cloning

    var queries = [ ];
    queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, xFrameOptions, lastBackupId ]
    });

    Object.keys(portBindings).forEach(function (env) {
@@ -283,6 +270,7 @@ function updateWithConstraints(id, app, constraints, callback) {
    assert.strictEqual(typeof constraints, 'string');
    assert.strictEqual(typeof callback, 'function');
    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');

    var queries = [ ];

@@ -301,9 +289,6 @@ function updateWithConstraints(id, app, constraints, callback) {
        if (p === 'manifest') {
            fields.push('manifestJson = ?');
            values.push(JSON.stringify(app[p]));
-        } else if (p === 'lastBackupConfig') {
-            fields.push('lastBackupConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
        } else if (p === 'oldConfig') {
            fields.push('oldConfigJson = ?');
            values.push(JSON.stringify(app[p]));
@@ -362,14 +347,17 @@ function setInstallationCommand(appId, installationState, values, callback) {
    // uninstall is allowed in any state
    // force update is allowed in any state including pending_uninstall! (for better or worse)
    // restore is allowed from installed or error state
-    // update and configure are allowed only in installed state
+    // configure is allowed in installed state or currently configuring or in error state
+    // update and backup are allowed only in installed state

    if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
        updateWithConstraints(appId, values, '', callback);
    } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_BACKUP) {
        updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else if (installationState === exports.ISTATE_PENDING_CONFIGURE) {
+        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "pending_configure" OR installationState = "error")', callback);
    } else {
        callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
    }
@@ -3,6 +3,7 @@
 var appdb = require('./appdb.js'),
    assert = require('assert'),
    async = require('async'),
+    config = require('./config.js'),
    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:apphealthmonitor'),
    docker = require('./docker.js').connection,
@@ -16,7 +17,7 @@ exports = module.exports = {
 };

 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
 var gHealthInfo = { }; // { time, emailSent }
 var gRunTimeout = null;
 var gDockerEventStream = null;
@@ -24,8 +25,11 @@ var gDockerEventStream = null;
 function debugApp(app) {
    assert(!app || typeof app === 'object');

-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
+    var manifestAppId = app ? app.manifest.id : '';
+    var id = app ? app.id : '';
+
+    debug(prefix + ' ' + manifestAppId + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + id);
 }

 function setHealth(app, health, callback) {
@@ -89,11 +93,14 @@ function checkAppHealth(app, callback) {
        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
        superagent
            .get(healthCheckUrl)
+            .set('Host', config.appFqdn(app.location)) // required for some apache configs with rewrite rules
            .redirects(0)
            .timeout(HEALTHCHECK_INTERVAL)
            .end(function (error, res) {
-
-            if (error || res.status >= 400) { // 2xx and 3xx are ok
+            if (error && !error.response) {
+                debugApp(app, 'not alive (network error): %s', error.message);
+                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
+            } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
                debugApp(app, 'not alive : %s', error || res.status);
                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
            } else {
@@ -112,7 +119,7 @@ function processApps(callback) {

            var alive = apps
                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
-                .map(function (a) { return a.location; }).join(', ');
+                .map(function (a) { return (a.location || 'naked_domain') + '|' + a.manifest.id; }).join(', ');

            debug('apps alive: [%s]', alive);

@@ -131,13 +138,16 @@ function run() {

 /*
    OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        docker run -ti -m 100M cloudron/base:0.8.1 /bin/bash
        apt-get update && apt-get install stress
        stress --vm 1 --vm-bytes 200M --vm-hang 0
 */
 function processDockerEvents() {
    // note that for some reason, the callback is called only on the first event
    debug('Listening for docker events');
+    const OOM_MAIL_LIMIT = 60 * 60 * 1000; // 60 minutes
+    var lastOomMailTime = new Date(new Date() - OOM_MAIL_LIMIT);
+
    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
        if (error) return console.error(error);

@@ -147,15 +157,19 @@ function processDockerEvents() {
        stream.on('data', function (data) {
            var ev = JSON.parse(data);
            debug('Container ' + ev.id + ' went OOM');
-            appdb.getByContainerId(ev.id, function (error, app) {
+            appdb.getByContainerId(ev.id, function (error, app) { // this can error for addons
                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
                var context = JSON.stringify(ev);
+                var now = new Date();
                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';

                debug('OOM Context: %s', context);

                // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+                if ((!app || app.appStoreId !== '') && (now - lastOomMailTime > OOM_MAIL_LIMIT)) {
+                    mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
+                    lastOomMailTime = now;
+                }
            });
        });

@@ -187,7 +201,7 @@ function stop(callback) {
    assert.strictEqual(typeof callback, 'function');

    clearTimeout(gRunTimeout);
-    gDockerEventStream.end();
+    if (gDockerEventStream) gDockerEventStream.end();

    callback();
 }
@@ -1,7 +1,5 @@
 #!/usr/bin/env node

-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -9,7 +7,7 @@ exports = module.exports = {
    startTask: startTask,

    // exported for testing
-    _getFreePort: getFreePort,
+    _reserveHttpPort: reserveHttpPort,
    _configureNginx: configureNginx,
    _unconfigureNginx: unconfigureNginx,
    _createVolume: createVolume,
@@ -19,8 +17,8 @@ exports = module.exports = {
    _verifyManifest: verifyManifest,
    _registerSubdomain: registerSubdomain,
    _unregisterSubdomain: unregisterSubdomain,
-    _reloadNginx: reloadNginx,
-    _waitForDnsPropagation: waitForDnsPropagation
+    _waitForDnsPropagation: waitForDnsPropagation,
+    _waitForAltDomainDnsPropagation: waitForAltDomainDnsPropagation
 };

 require('supererror')({ splatchError: true });
@@ -36,17 +34,19 @@ var addons = require('./addons.js'),
    apps = require('./apps.js'),
    assert = require('assert'),
    async = require('async'),
-    clientdb = require('./clientdb.js'),
+    backups = require('./backups.js'),
+    certificates = require('./certificates.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
+    ClientsError = clients.ClientsError,
    database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:apptask'),
    docker = require('./docker.js'),
    ejs = require('ejs'),
    fs = require('fs'),
-    hat = require('hat'),
    manifestFormat = require('cloudron-manifestformat'),
    net = require('net'),
+    nginx = require('./nginx.js'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
@@ -56,90 +56,67 @@ var addons = require('./addons.js'),
    superagent = require('superagent'),
    sysinfo = require('./sysinfo.js'),
    util = require('util'),
-    uuid = require('node-uuid'),
+    waitForDns = require('./waitfordns.js'),
    _ = require('underscore');

-var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
-    COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
-    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
+var COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
    RELOAD_COLLECTD_CMD = path.join(__dirname, 'scripts/reloadcollectd.sh'),
    RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh'),
    CREATEAPPDIR_CMD = path.join(__dirname, 'scripts/createappdir.sh');

 function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
    database.initialize(callback);
 }

-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
+function debugApp(app) {
+    assert.strictEqual(typeof app, 'object');

    var prefix = app ? (app.location || '(bare)') : '(no app)';
    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }

-// We expect conflicts to not happen despite closing the port (parallel app installs, app update does not reconfigure nginx etc)
-// https://tools.ietf.org/html/rfc6056#section-3.5 says linux uses random ephemeral port allocation
-function getFreePort(callback) {
+function reserveHttpPort(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    var server = net.createServer();
    server.listen(0, function () {
        var port = server.address().port;
-        server.close(function () {
-            return callback(null, port);
+        updateApp(app, { httpPort: port }, function (error) {
+            if (error) {
+                server.close();
+                return callback(error);
+            }
+
+            server.close(callback);
        });
    });
 }

-function reloadNginx(callback) {
-    shell.sudo('reloadNginx', [ RELOAD_NGINX_CMD ], callback);
-}
-
 function configureNginx(app, callback) {
-    getFreePort(function (error, freePort) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    certificates.ensureCertificate(app, function (error, certFilePath, keyFilePath) {
        if (error) return callback(error);

-        var sourceDir = path.resolve(__dirname, '..');
-        var endpoint = app.oauthProxy ? 'oauthproxy' : 'app';
-        var vhost = config.appFqdn(app.location);
-        var certFilePath = safe.fs.statSync(path.join(paths.APP_CERTS_DIR, vhost + '.cert')) ? path.join(paths.APP_CERTS_DIR, vhost + '.cert') : 'cert/host.cert';
-        var keyFilePath = safe.fs.statSync(path.join(paths.APP_CERTS_DIR, vhost + '.key')) ? path.join(paths.APP_CERTS_DIR, vhost + '.key') : 'cert/host.key';
-
-        var data = {
-            sourceDir: sourceDir,
-            adminOrigin: config.adminOrigin(),
-            vhost: vhost,
-            port: freePort,
-            endpoint: endpoint,
-            certFilePath: certFilePath,
-            keyFilePath: keyFilePath
-        };
-        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
-
-        var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-        debugApp(app, 'writing config to %s', nginxConfigFilename);
-
-        if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-            debugApp(app, 'Error creating nginx config : %s', safe.error.message);
-            return callback(safe.error);
-        }
-
-        async.series([
-            exports._reloadNginx,
-            updateApp.bind(null, app, { httpPort: freePort })
-        ], callback);
+        nginx.configureApp(app, certFilePath, keyFilePath, callback);
    });
 }

 function unconfigureNginx(app, callback) {
-    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    if (!safe.fs.unlinkSync(nginxConfigFilename)) {
-        debugApp(app, 'Error removing nginx configuration : %s', safe.error.message);
-        return callback(null);
-    }
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');

-    exports._reloadNginx(callback);
+    // TODO: maybe revoke the cert
+    nginx.unconfigureApp(app, callback);
 }

 function createContainer(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
    assert(!app.containerId); // otherwise, it will trigger volumeFrom

    debugApp(app, 'creating container');
@@ -152,6 +129,9 @@ function createContainer(app, callback) {
 }

 function deleteContainers(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    debugApp(app, 'deleting containers');

    docker.deleteContainers(app.id, function (error) {
@@ -162,10 +142,16 @@ function deleteContainers(app, callback) {
 }

 function createVolume(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    shell.sudo('createVolume', [ CREATEAPPDIR_CMD, app.id ], callback);
 }

 function deleteVolume(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    shell.sudo('deleteVolume', [ RMAPPDIR_CMD, app.id ], callback);
 }

@@ -173,22 +159,20 @@ function allocateOAuthProxyCredentials(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    if (!app.oauthProxy) return callback(null);
+    if (!nginx.requiresOAuthProxy(app)) return callback(null);

-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
    var redirectURI = 'https://' + config.appFqdn(app.location);
    var scope = 'profile';

-    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
 }

 function removeOAuthProxyCredentials(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) {
            debugApp(app, 'Error removing OAuth client id', error);
            return callback(error);
        }
@@ -198,6 +182,9 @@ function removeOAuthProxyCredentials(app, callback) {
 }

 function addCollectdProfile(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    var collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId });
    fs.writeFile(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), collectdConf, function (error) {
        if (error) return callback(error);
@@ -206,6 +193,9 @@ function addCollectdProfile(app, callback) {
 }

 function removeCollectdProfile(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), function (error) {
        if (error && error.code !== 'ENOENT') debugApp(app, 'Error removing collectd profile', error);
        shell.sudo('removeCollectdProfile', [ RELOAD_COLLECTD_CMD ], callback);
@@ -213,6 +203,9 @@ function removeCollectdProfile(app, callback) {
 }

 function verifyManifest(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    debugApp(app, 'Verifying manifest');

    var manifest = app.manifest;
@@ -226,64 +219,87 @@ function verifyManifest(app, callback) {
 }

 function downloadIcon(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    debugApp(app, 'Downloading icon of %s@%s', app.appStoreId, app.manifest.version);

    var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';

-    superagent
-        .get(iconUrl)
-        .buffer(true)
-        .end(function (error, res) {
-            if (error) return callback(new Error('Error downloading icon:' + error.message));
-            if (res.status !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        superagent
+            .get(iconUrl)
+            .buffer(true)
+            .end(function (error, res) {
+                if (error && !error.response) return retryCallback(new Error('Network error downloading icon:' + error.message));
+                if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli

-            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return retryCallback(new Error('Error saving icon:' + safe.error.message));

-            callback(null);
-    });
+                retryCallback(null);
+        });
+    }, callback);
 }

 function registerSubdomain(app, callback) {
-    // even though the bare domain is already registered in the appstore, we still
-    // need to register it so that we have a dnsRecordId to wait for it to complete
-    async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
-        debugApp(app, 'Registering subdomain location [%s]', app.location);
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');

-        subdomains.add(app.location, 'A', [ sysinfo.getIp() ], function (error, changeId) {
-            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+    sysinfo.getIp(function (error, ip) {
+        if (error) return callback(error);

-            retryCallback(null, error || changeId);
+        // even though the bare domain is already registered in the appstore, we still
+        // need to register it so that we have a dnsRecordId to wait for it to complete
+        async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+            debugApp(app, 'Registering subdomain location [%s]', app.location);
+
+            subdomains.add(app.location, 'A', [ ip ], function (error, changeId) {
+                if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+                retryCallback(null, error || changeId);
+            });
+        }, function (error, result) {
+            if (error || result instanceof Error) return callback(error || result);
+
+            updateApp(app, { dnsRecordId: result }, callback);
        });
-    }, function (error, result) {
-        if (error || result instanceof Error) return callback(error || result);
-
-        updateApp(app, { dnsRecordId: result }, callback);
    });
 }

 function unregisterSubdomain(app, location, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
    // do not unregister bare domain because we show a error/cloudron info page there
    if (location === '') {
        debugApp(app, 'Skip unregister of empty subdomain');
        return callback(null);
    }

-    async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
-        debugApp(app, 'Unregistering subdomain: %s', location);
+    sysinfo.getIp(function (error, ip) {
+        if (error) return callback(error);

-        subdomains.remove(location, 'A', [ sysinfo.getIp() ], function (error) {
-            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+        async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+            debugApp(app, 'Unregistering subdomain: %s', location);

-            retryCallback(null, error);
+            subdomains.remove(location, 'A', [ ip ], function (error) {
+                if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+                retryCallback(null, error);
+            });
+        }, function (error, result) {
+            if (error || result instanceof Error) return callback(error || result);
+
+            updateApp(app, { dnsRecordId: null }, callback);
        });
-    }, function (error, result) {
-        if (error || result instanceof Error) return callback(error || result);
-
-        updateApp(app, { dnsRecordId: null }, callback);
    });
 }

 function removeIcon(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    fs.unlink(path.join(paths.APPICONS_DIR, app.id + '.png'), function (error) {
        if (error && error.code !== 'ENOENT') debugApp(app, 'cannot remove icon : %s', error);
        callback(null);
@@ -291,29 +307,41 @@ function removeIcon(app, callback) {
 }

 function waitForDnsPropagation(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    if (!config.CLOUDRON) {
        debugApp(app, 'Skipping dns propagation check for development');
        return callback(null);
    }

-    function retry(error) {
-        debugApp(app, 'waitForDnsPropagation: ', error);
-        setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
-    }
+    async.retry({ interval: 5000, times: 120 }, function checkStatus(retryCallback) {
+        subdomains.status(app.dnsRecordId, function (error, result) {
+            if (error) return retryCallback(new Error('Failed to get dns record status : ' + error.message));

-    subdomains.status(app.dnsRecordId, function (error, result) {
-        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);

-        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
+            if (result !== 'done') return retryCallback(new Error(util.format('app:%s not ready yet: %s', app.id, result)));

-        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
+            retryCallback(null, result);
+        });
+    }, callback);
+}

-        callback(null);
-    });
+function waitForAltDomainDnsPropagation(app, callback) {
+    if (!app.altDomain) return callback(null);
+
+    // try for 10 minutes before giving up. this allows the user to "reconfigure" the app in the case where
+    // an app has an external domain and cloudron is migrated to custom domain.
+    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', { interval: 10000, times: 60 }, callback);
 }

 // updates the app object and the database
 function updateApp(app, values, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof values, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    debugApp(app, 'updating app with values: %j', values);

    appdb.update(app.id, values, function (error) {
@@ -338,11 +366,15 @@ function updateApp(app, values, callback) {
 //   - setup the container (requires image, volumes, addons)
 //   - setup collectd (requires container id)
 function install(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    async.series([
        verifyManifest.bind(null, app),

        // teardown for re-installs
        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        unconfigureNginx.bind(null, app),
        removeCollectdProfile.bind(null, app),
        stopApp.bind(null, app),
        deleteContainers.bind(null, app),
@@ -351,10 +383,8 @@ function install(app, callback) {
        unregisterSubdomain.bind(null, app, app.location),
        removeOAuthProxyCredentials.bind(null, app),
        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
-        unconfigureNginx.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
-        configureNginx.bind(null, app),
+        reserveHttpPort.bind(null, app),

        updateApp.bind(null, app, { installationProgress: '20, Downloading icon' }),
        downloadIcon.bind(null, app),
@@ -382,9 +412,15 @@ function install(app, callback) {

        runApp.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
        exports._waitForDnsPropagation.bind(null, app),

+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
+
+        updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
+        configureNginx.bind(null, app),
+
        // done!
        function (callback) {
            debugApp(app, 'installed');
@@ -400,9 +436,12 @@ function install(app, callback) {
 }

 function backup(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    async.series([
        updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        apps.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest),

        // done!
        function (callback) {
@@ -420,6 +459,9 @@ function backup(app, callback) {

 // restore is also called for upgrades and infra updates. note that in those cases it is possible there is no backup
 function restore(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    // we don't have a backup, same as re-install. this allows us to install from install failures (update failures always
    // have a backupId)
    if (!app.lastBackupId) {
@@ -427,8 +469,11 @@ function restore(app, callback) {
        return install(app, callback);
    }

+    var backupId = app.lastBackupId;
+
    async.series([
        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        unconfigureNginx.bind(null, app),
        removeCollectdProfile.bind(null, app),
        stopApp.bind(null, app),
        deleteContainers.bind(null, app),
@@ -442,10 +487,8 @@ function restore(app, callback) {
        },
        removeOAuthProxyCredentials.bind(null, app),
        removeIcon.bind(null, app),
-        unconfigureNginx.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '30, Configuring Nginx' }),
-        configureNginx.bind(null, app),
+        reserveHttpPort.bind(null, app),

        updateApp.bind(null, app, { installationProgress: '40, Downloading icon' }),
        downloadIcon.bind(null, app),
@@ -463,7 +506,7 @@ function restore(app, callback) {
        createVolume.bind(null, app),

        updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons),
+        backups.restoreApp.bind(null, app, app.manifest.addons, backupId),

        updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
        createContainer.bind(null, app),
@@ -473,9 +516,15 @@ function restore(app, callback) {

        runApp.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
        exports._waitForDnsPropagation.bind(null, app),

+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
+        configureNginx.bind(null, app),
+
        // done!
        function (callback) {
            debugApp(app, 'restored');
@@ -493,8 +542,12 @@ function restore(app, callback) {

 // note that configure is called after an infra update as well
 function configure(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    async.series([
        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        unconfigureNginx.bind(null, app),
        removeCollectdProfile.bind(null, app),
        stopApp.bind(null, app),
        deleteContainers.bind(null, app),
@@ -504,10 +557,8 @@ function configure(app, callback) {
            unregisterSubdomain(app, app.oldConfig.location, next);
        },
        removeOAuthProxyCredentials.bind(null, app),
-        unconfigureNginx.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '25, Configuring Nginx' }),
-        configureNginx.bind(null, app),
+        reserveHttpPort.bind(null, app),

        updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
        allocateOAuthProxyCredentials.bind(null, app),
@@ -530,6 +581,12 @@ function configure(app, callback) {
        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
        exports._waitForDnsPropagation.bind(null, app),

+        updateApp.bind(null, app, { installationProgress: '85, Waiting for External Domain CNAME setup' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
+        configureNginx.bind(null, app),
+
        // done!
        function (callback) {
            debugApp(app, 'configured');
@@ -546,22 +603,30 @@ function configure(app, callback) {

 // nginx configuration is skipped because app.httpPort is expected to be available
 function update(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    debugApp(app, 'Updating to %s', safe.query(app, 'manifest.version'));

    // app does not want these addons anymore
+    // FIXME: this does not handle option changes (like multipleDatabases)
    var unusedAddons = _.omit(app.oldConfig.manifest.addons, Object.keys(app.manifest.addons));

    async.series([
        updateApp.bind(null, app, { installationProgress: '0, Verify manifest' }),
        verifyManifest.bind(null, app),

+        // download new image before app is stopped. this is so we can reduce downtime
+        // and also not remove the 'common' layers when the old image is deleted
+        updateApp.bind(null, app, { installationProgress: '15, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
        // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
        // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
-        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        updateApp.bind(null, app, { installationProgress: '25, Cleaning up old install' }),
        removeCollectdProfile.bind(null, app),
        stopApp.bind(null, app),
        deleteContainers.bind(null, app),
-        addons.teardownAddons.bind(null, app, unusedAddons),
        function deleteImageIfChanged(done) {
             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();

@@ -573,16 +638,16 @@ function update(app, callback) {
            if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);

            async.series([
-                updateApp.bind(null, app, { installationProgress: '20, Backup app' }),
-                apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
+                backups.backupApp.bind(null, app, app.oldConfig.manifest)
            ], next);
        },

-        updateApp.bind(null, app, { installationProgress: '35, Downloading icon' }),
-        downloadIcon.bind(null, app),
+        // only delete unused addons after backup
+        addons.teardownAddons.bind(null, app, unusedAddons),

-        updateApp.bind(null, app, { installationProgress: '45, Downloading image' }),
-        docker.downloadImage.bind(null, app.manifest),
+        updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
+        downloadIcon.bind(null, app),

        updateApp.bind(null, app, { installationProgress: '70, Updating addons' }),
        addons.setupAddons.bind(null, app, app.manifest.addons),
@@ -610,6 +675,9 @@ function update(app, callback) {
 }

 function uninstall(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    debugApp(app, 'uninstalling');

    async.series([
@@ -645,10 +713,19 @@ function uninstall(app, callback) {

        updateApp.bind(null, app, { installationProgress: '95, Remove app from database' }),
        appdb.del.bind(null, app.id)
-    ], callback);
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error uninstalling app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
 }

 function runApp(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    docker.startContainer(app.containerId, function (error) {
        if (error) return callback(error);

@@ -657,6 +734,9 @@ function runApp(app, callback) {
 }

 function stopApp(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    docker.stopContainers(app.id, function (error) {
        if (error) return callback(error);

@@ -665,6 +745,9 @@ function stopApp(app, callback) {
 }

 function handleRunCommand(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    if (app.runState === appdb.RSTATE_PENDING_STOP) {
        return stopApp(app, callback);
    }
@@ -680,6 +763,9 @@ function handleRunCommand(app, callback) {
 }

 function startTask(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
    // determine what to do
    appdb.get(appId, function (error, app) {
        if (error) return callback(error);
@@ -694,9 +780,10 @@ function startTask(appId, callback) {
        case appdb.ISTATE_PENDING_BACKUP: return backup(app, callback);
        case appdb.ISTATE_INSTALLED: return handleRunCommand(app, callback);
        case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
+        case appdb.ISTATE_PENDING_CLONE: return restore(app, callback);
        case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
        case appdb.ISTATE_ERROR:
-            debugApp(app, 'Apptask launched with error states.');
+            debugApp(app, 'Internal error. apptask launched with error status.');
            return callback(null);
        default:
            debugApp(app, 'apptask launched with invalid command');
@@ -710,6 +797,11 @@ if (require.main === module) {

    debug('Apptask for %s', process.argv[2]);

+    process.on('SIGTERM', function () {
+        debug('taskmanager sent SIGTERM since it got a new task for this app');
+        process.exit(0);
+    });
+
    initialize(function (error) {
        if (error) throw error;

@@ -723,4 +815,3 @@ if (require.main === module) {
        });
    });
 }
-
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -10,8 +8,9 @@ exports = module.exports = {
 var assert = require('assert'),
    BasicStrategy = require('passport-http').BasicStrategy,
    BearerStrategy = require('passport-http-bearer').Strategy,
-    clientdb = require('./clientdb'),
+    clients = require('./clients'),
    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    ClientsError = clients.ClientsError,
    DatabaseError = require('./databaseerror'),
    debug = require('debug')('box:auth'),
    LocalStrategy = require('passport-local').Strategy,
@@ -19,7 +18,6 @@ var assert = require('assert'),
    passport = require('passport'),
    tokendb = require('./tokendb'),
    user = require('./user'),
-    userdb = require('./userdb'),
    UserError = user.UserError,
    _ = require('underscore');

@@ -27,11 +25,11 @@ function initialize(callback) {
    assert.strictEqual(typeof callback, 'function');

    passport.serializeUser(function (user, callback) {
-        callback(null, user.username);
+        callback(null, user.id);
    });

-    passport.deserializeUser(function(username, callback) {
-        userdb.get(username, function (error, result) {
+    passport.deserializeUser(function(userId, callback) {
+        user.get(userId, function (error, result) {
            if (error) return callback(error);

            var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -43,7 +41,7 @@ function initialize(callback) {

    passport.use(new LocalStrategy(function (username, password, callback) {
        if (username.indexOf('@') === -1) {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                if (error) return callback(error);
@@ -66,14 +64,14 @@ function initialize(callback) {
            debug('BasicStrategy: detected client id %s instead of username:password', username);
            // username is actually client id here
            // password is client secret
-            clientdb.get(username, function (error, client) {
-                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            clients.get(username, function (error, client) {
+                if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
                if (error) return callback(error);
                if (client.clientSecret != password) return callback(null, false);
                return callback(null, client);
            });
        } else {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                if (error) return callback(error);
@@ -84,8 +82,8 @@ function initialize(callback) {
    }));

    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function(error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
            if (error) { return callback(error); }
            if (client.clientSecret != clientSecret) { return callback(null, false); }
            return callback(null, client);
@@ -100,29 +98,11 @@ function initialize(callback) {
            // scopes here can define what capabilities that token carries
            // passport put the 'info' object into req.authInfo, where we can further validate the scopes
            var info = { scope: token.scope };
-            var tokenType;

-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }
-
-            userdb.get(token.identifier, function (error, user) {
+            user.get(token.identifier, function (error, user) {
                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                if (error) return callback(error);

-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
                callback(null, user, info);
            });
        });
@@ -136,4 +116,3 @@ function uninitialize(callback) {

    callback(null);
 }
-
@@ -0,0 +1,114 @@
+'use strict';
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', ];
+
+exports = module.exports = {
+    add: add,
+    getPaged: getPaged,
+    get: get,
+    del: del,
+    getByAppIdPaged: getByAppIdPaged,
+
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+}
+
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, 'appbackup\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
+        [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function add(backup, callback) {
+    assert(backup && typeof backup === 'object');
+    assert.strictEqual(typeof backup.id, 'string');
+    assert.strictEqual(typeof backup.version, 'string');
+    assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
+    assert(util.isArray(backup.dependsOn));
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = backup.creationTime || new Date(); // allow tests to set the time
+
+    database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn) VALUES (?, ?, ?, ?, ?, ?)',
+        [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(',') ],
+        function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
@@ -3,21 +3,56 @@
 exports = module.exports = {
    BackupsError: BackupsError,

-    getAllPaged: getAllPaged,
+    getPaged: getPaged,
+    getByAppIdPaged: getByAppIdPaged,

-    getBackupUrl: getBackupUrl,
    getRestoreUrl: getRestoreUrl,
+    getRestoreConfig: getRestoreConfig,

-    copyLastBackup: copyLastBackup
+    ensureBackup: ensureBackup,
+
+    backup: backup,
+    backupApp: backupApp,
+    restoreApp: restoreApp,
+
+    backupBoxAndApps: backupBoxAndApps
 };

-var assert = require('assert'),
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    async = require('async'),
+    assert = require('assert'),
+    backupdb = require('./backupdb.js'),
    caas = require('./storage/caas.js'),
    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:backups'),
+    eventlog = require('./eventlog.js'),
+    locker = require('./locker.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    progress = require('./progress.js'),
    s3 = require('./storage/s3.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
    settings = require('./settings.js'),
-    util = require('util');
+    superagent = require('superagent'),
+    util = require('util'),
+    webhooks = require('./webhooks.js');
+
+var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}

 function BackupsError(reason, errorOrMessage) {
    assert.strictEqual(typeof reason, 'string');
@@ -40,6 +75,7 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.BAD_STATE = 'bad state';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';

 // choose which storage backend we use for test purpose we use s3
@@ -51,49 +87,105 @@ function api(provider) {
    }
 }

-function getAllPaged(page, perPage, callback) {
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
    assert.strictEqual(typeof callback, 'function');

+    backupdb.getPaged(page, perPage, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    backupdb.getByAppIdPaged(page, perPage, appId, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getBoxBackupCredentials(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
+    var filename = filebase + '.tar.gz';
+
    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
-            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
+            if (error) return callback(error);

-            return callback(null, backups); // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first
+            result.id = filename;
+            result.s3Url = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + filename;
+            result.backupKey = backupConfig.key;
+
+            debug('getBoxBackupCredentials: %j', result);
+
+            callback(null, result);
        });
    });
 }

-function getBackupUrl(app, callback) {
-    assert(!app || typeof app === 'object');
+function getAppBackupCredentials(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
    assert.strictEqual(typeof callback, 'function');

-    var filename = '';
-    if (app) {
-        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    } else {
-        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
-    }
+    var now = new Date();
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
+    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';

    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
            if (error) return callback(error);

-            var obj = {
-                id: filename,
-                url: result.url,
-                sessionToken: result.sessionToken,
-                backupKey: backupConfig.key
-            };
+            result.id = dataFilename;
+            result.s3ConfigUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + configFilename;
+            result.s3DataUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + dataFilename;
+            result.backupKey = backupConfig.key;

-            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getAppBackupCredentials: %j', result);

-            callback(null, obj);
+            callback(null, result);
+        });
+    });
+}
+
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+function getRestoreConfig(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var configFile = backupId.replace(/\.tar\.gz$/, '.json');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getRestoreUrl(backupConfig, configFile, function (error, result) {
+            if (error) return callback(error);
+
+            superagent.get(result.url).buffer(true).end(function (error, response) {
+                if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+                if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
+
+                var config = safe.JSON.parse(response.text);
+                if (!config) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error in config:' + safe.error.message));
+
+                return callback(null, config);
+            });
        });
    });
 }
@@ -106,37 +198,300 @@ function getRestoreUrl(backupId, callback) {
    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
+        api(backupConfig.provider).getRestoreUrl(backupConfig, backupId, function (error, result) {
            if (error) return callback(error);

            var obj = {
                id: backupId,
                url: result.url,
-                sessionToken: result.sessionToken,
                backupKey: backupConfig.key
            };

-            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getRestoreUrl: id:%s url:%s backupKey:%s', obj.id, obj.url, obj.backupKey);

            callback(null, obj);
        });
    });
 }

-function copyLastBackup(app, callback) {
-    assert(app && typeof app === 'object');
+function copyLastBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
    assert.strictEqual(typeof callback, 'function');

-    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    var now = new Date();
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), manifest.version);

    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilename, function (error) {
+        debug('copyLastBackup: copying archive %s to %s', app.lastBackupId, toFilenameArchive);
+
+        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));

-            return callback(null, toFilename);
+            // TODO change that logic by adjusting app.lastBackupId to not contain the file type
+            var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';
+
+            debug('copyLastBackup: copying config %s to %s', configFileId, toFilenameConfig);
+
+            api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+                return callback(null, toFilenameArchive);
+            });
+        });
+    });
+}
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    getBoxBackupCredentials(appBackupIds, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        debug('backupBoxWithAppBackupIds:  %j', result);
+
+        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
+        if (result.sessionToken) args.push(result.sessionToken);
+
+        shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debug('backupBoxWithAppBackupIds: success');
+
+            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                    if (error) return callback(error);
+                    callback(null, result.id);
+                });
+            });
+        });
+    });
+}
+
+// this function expects you to have a lock
+// function backupBox(callback) {
+//    apps.getAll(function (error, allApps) {
+//         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+//
+//         var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+//         appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+//
+//         backupBoxWithAppBackupIds(appBackupIds, callback);
+//     });
+// }
+
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldAppBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    copyLastBackup(app, manifest, function (error, newBackupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewAppBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAppBackupCredentials(app, manifest, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
+
+        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey, result.region, result.backupKey ];
+        if (result.sessionToken) args.push(result.sessionToken);
+
+        async.series([
+            addons.backupAddons.bind(null, app, manifest.addons),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
+        ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debugApp(app, 'createNewAppBackup: %s done', result.id);
+
+            backupdb.add({ id: result.id, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, result.id);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function backupApp(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        backupFunction = reuseOldAppBackup.bind(null, app, manifest);
+    } else {
+        var appConfig = apps.getAppConfig(app);
+        appConfig.manifest = manifest;
+        backupFunction = createNewAppBackup.bind(null, app, manifest);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, function (error) {
+            if (error) return callback(error);
+
+            return callback(null, backupId);
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    callback = callback || NOOP_CALLBACK;
+
+    eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { });
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            backupApp(app, app.manifest, function (error, backupId) {
+                if (error && error.reason !== BackupsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
+                }
+
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+
+                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, filename: filename });
+
+                callback(error, filename);
+            });
+        });
+    });
+}
+
+function backup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
+
+    progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
+
+    backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background
+        if (error) debug('backup failed.', error);
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    getPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        backup(auditSource, callback);
+    });
+}
+
+function restoreApp(app, addonsToRestore, backupId, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    getRestoreUrl(backupId, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
        });
    });
 }
@@ -0,0 +1,470 @@
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:cert/acme'),
+    fs = require('fs'),
+    parseLinks = require('parse-links'),
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
+    superagent = require('superagent'),
+    ursa = require('ursa'),
+    util = require('util'),
+    _ = require('underscore');
+
+var CA_PROD = 'https://acme-v01.api.letsencrypt.org',
+    CA_STAGING = 'https://acme-staging.api.letsencrypt.org',
+    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'acme'
+};
+
+function AcmeError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AcmeError, Error);
+AcmeError.INTERNAL_ERROR = 'Internal Error';
+AcmeError.EXTERNAL_ERROR = 'External Error';
+AcmeError.ALREADY_EXISTS = 'Already Exists';
+AcmeError.NOT_COMPLETED = 'Not Completed';
+AcmeError.FORBIDDEN = 'Forbidden';
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function Acme(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    this.caOrigin = options.prod ? CA_PROD : CA_STAGING;
+    this.accountKeyPem = null; // Buffer
+    this.email = options.email;
+}
+
+Acme.prototype.getNonce = function (callback) {
+    superagent.get(this.caOrigin + '/directory', function (error, response) {
+        if (error) return callback(error);
+        if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
+
+        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
+    });
+};
+
+// urlsafe base64 encoding (jose)
+function urlBase64Encode(string) {
+    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+}
+
+function b64(str) {
+    var buf = util.isBuffer(str) ? str : new Buffer(str);
+   return urlBase64Encode(buf.toString('base64'));
+}
+
+Acme.prototype.sendSignedRequest = function (url, payload, callback) {
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof payload, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    assert(util.isBuffer(this.accountKeyPem));
+    var privateKey = ursa.createPrivateKey(this.accountKeyPem);
+
+    var header = {
+        alg: 'RS256',
+        jwk: {
+            e: b64(privateKey.getExponent()),
+            kty: 'RSA',
+            n: b64(privateKey.getModulus())
+        }
+    };
+ 
+    var payload64 = b64(payload);
+
+    this.getNonce(function (error, nonce) {
+        if (error) return callback(error);
+
+        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
+
+        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+        var signer = ursa.createSigner('sha256');
+        signer.update(protected64 + '.' + payload64, 'utf8');
+        var signature64 = urlBase64Encode(signer.sign(privateKey, 'base64'));
+
+        var data = {
+            header: header,
+            protected: protected64,
+            payload: payload64,
+            signature: signature64
+        };
+
+        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).end(function (error, res) {
+            if (error && !error.response) return callback(error); // network errors
+
+            callback(null, res);
+        });
+    });
+};
+
+Acme.prototype.updateContact = function (registrationUri, callback) {
+    assert.strictEqual(typeof registrationUri, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('updateContact: %s %s', registrationUri, this.email);
+
+    // https://github.com/ietf-wg-acme/acme/issues/30
+    var payload = {
+        resource: 'reg',
+        contact: [ 'mailto:' + this.email ],
+        agreement: LE_AGREEMENT
+    };
+
+    var that = this;
+    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
+        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 202, got %s %s', result.statusCode, result.text)));
+
+        debug('updateContact: contact of user updated to %s', that.email);
+
+        callback();
+    });
+};
+
+Acme.prototype.registerUser = function (callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-reg',
+        contact: [ 'mailto:' + this.email ],
+        agreement: LE_AGREEMENT
+    };
+
+    debug('registerUser: %s', this.email);
+
+    var that = this;
+    this.sendSignedRequest(this.caOrigin + '/acme/new-reg', JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
+        if (result.statusCode === 409) return that.updateContact(result.headers.location, callback); // already exists
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerUser: registered user %s', that.email);
+
+        callback(null);
+    });
+};
+
+Acme.prototype.registerDomain = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-authz',
+        identifier: {
+            type: 'dns',
+            value: domain
+        }
+    };
+
+    debug('registerDomain: %s', domain);
+
+    this.sendSignedRequest(this.caOrigin + '/acme/new-authz', JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering domain: ' + error.message));
+        if (result.statusCode === 403) return callback(new AcmeError(AcmeError.FORBIDDEN, result.body.detail));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerDomain: registered %s', domain);
+
+        callback(null, result.body);
+    });
+};
+
+Acme.prototype.prepareHttpChallenge = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    var token = challenge.token;
+
+    assert(util.isBuffer(this.accountKeyPem));
+    var privateKey = ursa.createPrivateKey(this.accountKeyPem);
+
+    var jwk = {
+        e: b64(privateKey.getExponent()),
+        kty: 'RSA',
+        n: b64(privateKey.getModulus())
+    };
+
+    var shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    var thumbprint = urlBase64Encode(shasum.digest('base64'));
+    var keyAuthorization = token + '.' + thumbprint;
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, token));
+
+    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, token), token + '.' + thumbprint, function (error) {
+        if (error) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, error));
+
+        callback();
+    });
+};
+
+Acme.prototype.notifyChallengeReady = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('notifyChallengeReady: %s was met', challenge.uri);
+
+    var keyAuthorization = fs.readFileSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), 'utf8');
+
+    var payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    this.sendSignedRequest(challenge.uri, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when notifying challenge: ' + error.message));
+        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 202, got %s %s', result.statusCode, result.text)));
+
+        callback();
+    });
+};
+
+Acme.prototype.waitForChallenge = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('waitingForChallenge: %j', challenge);
+
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        debug('waitingForChallenge: getting status');
+
+        superagent.get(challenge.uri).end(function (error, result) {
+            if (error && !error.response) {
+                debug('waitForChallenge: network error getting uri %s', challenge.uri);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, error.message)); // network error
+            }
+            if (result.statusCode !== 202) {
+                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+            }
+
+            debug('waitForChallenge: status is "%s"', result.body.status);
+
+            if (result.body.status === 'pending') return retryCallback(new AcmeError(AcmeError.NOT_COMPLETED));
+            else if (result.body.status === 'valid') return retryCallback();
+            else return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
+        });
+    }, function retryFinished(error) {
+        // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
+        callback(error);
+    });
+};
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+Acme.prototype.signCertificate = function (domain, csrDer, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert(util.isBuffer(csrDer));
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+
+    var payload = {
+        resource: 'new-cert',
+        csr: b64(csrDer)
+    };
+
+    debug('signCertificate: sending new-cert request');
+
+    this.sendSignedRequest(this.caOrigin + '/acme/new-cert', JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when signing certificate: ' + error.message));
+        // 429 means we reached the cert limit for this domain
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        var certUrl = result.headers.location;
+
+        if (!certUrl) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Missing location in downloadCertificate'));
+
+        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // maybe use for renewal
+
+        return callback(null, result.headers.location);
+    });
+};
+
+Acme.prototype.createKeyAndCsr = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+    var csrFile = path.join(outdir, domain + '.csr');
+    var privateKeyFile = path.join(outdir, domain + '.key');
+    var execSync = safe.child_process.execSync;
+
+    if (safe.fs.existsSync(privateKeyFile)) {
+        // in some old releases, csr file was corrupt. so always regenerate it
+        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
+    } else {
+        var key = execSync('openssl genrsa 4096');
+        if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+    }
+
+    var csrDer = execSync(util.format('openssl req -new -key %s -outform DER -subj /CN=%s', privateKeyFile, domain));
+    if (!csrDer) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error)); // bookkeeping
+
+    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
+
+    callback(null, csrDer);
+};
+
+// TODO: download the chain in a loop following 'up' header
+Acme.prototype.downloadChain = function (linkHeader, callback) {
+    if (!linkHeader) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Empty link header when downloading certificate chain');
+
+    var linkInfo = parseLinks(linkHeader);
+    if (!linkInfo || !linkInfo.up) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Failed to parse link header when downloading certificate chain'); 
+
+    debug('downloadChain: downloading from %s', this.caOrigin + linkInfo.up);
+
+    superagent.get(this.caOrigin + linkInfo.up).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var chainDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        var chainPem = execSync('openssl x509 -inform DER -outform PEM', { input: chainDer }); // this is really just base64 encoding with header
+        if (!chainPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        callback(null, chainPem);
+    });
+};
+
+Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof certUrl, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+    var that = this;
+
+    superagent.get(certUrl).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode === 202) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, 'Retry not implemented yet'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var certificateDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        safe.fs.writeFileSync(path.join(outdir, domain + '.der'), certificateDer);
+        debug('downloadCertificate: cert der file for %s saved', domain);
+
+        var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { input: certificateDer }); // this is really just base64 encoding with header
+        if (!certificatePem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        that.downloadChain(result.header['link'], function (error, chainPem) {
+            if (error) return callback(error);
+
+            var certificateFile = path.join(outdir, domain + '.cert');
+            var fullChainPem = Buffer.concat([certificatePem, chainPem]);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+            debug('downloadCertificate: cert file for %s saved at %s', domain, certificateFile);
+
+            callback();
+        });
+    });
+};
+
+Acme.prototype.acmeFlow = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
+        debug('getCertificate: generating acme account key on first run');
+        this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
+        if (!this.accountKeyPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
+    } else {
+        debug('getCertificate: using existing acme account key');
+        this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
+    }
+
+    var that = this;
+    this.registerUser(function (error) {
+        if (error) return callback(error);
+
+        that.registerDomain(domain, function (error, result) {
+            if (error) return callback(error);
+
+            debug('acmeFlow: challenges: %j', result);
+
+            var httpChallenges = result.challenges.filter(function(x) { return x.type === 'http-01'; });
+            if (httpChallenges.length === 0) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'no http challenges'));
+            var challenge = httpChallenges[0];
+
+            async.waterfall([
+                that.prepareHttpChallenge.bind(that, challenge),
+                that.notifyChallengeReady.bind(that, challenge),
+                that.waitForChallenge.bind(that, challenge),
+                that.createKeyAndCsr.bind(that, domain),
+                that.signCertificate.bind(that, domain),
+                that.downloadCertificate.bind(that, domain)
+            ], callback);
+        });
+    });
+};
+
+Acme.prototype.getCertificate = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
+    this.acmeFlow(domain, function (error) {
+        if (error) return callback(error);
+
+        var outdir = paths.APP_CERTS_DIR;
+        callback(null, path.join(outdir, domain + '.cert'), path.join(outdir, domain + '.key'));
+    });
+};
+
+function getCertificate(domain, options, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var acme = new Acme(options || { });
+    acme.getCertificate(domain, callback);
+}
@@ -0,0 +1,21 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'caas'
+};
+
+var assert = require('assert'),
+	debug = require('debug')('box:cert/caas.js');
+
+function getCertificate(domain, options, callback) {
+	assert.strictEqual(typeof domain, 'string');
+	assert.strictEqual(typeof options, 'object');
+	assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: using fallback certificate', domain);
+
+    return callback(null, 'cert/host.cert', 'cert/host.key');
+}
@@ -0,0 +1,341 @@
+'use strict';
+
+exports = module.exports = {
+    installAdminCertificate: installAdminCertificate,
+    renewAll: renewAll,
+    setFallbackCertificate: setFallbackCertificate,
+    setAdminCertificate: setAdminCertificate,
+    CertificatesError: CertificatesError,
+    validateCertificate: validateCertificate,
+    ensureCertificate: ensureCertificate,
+    getAdminCertificatePath: getAdminCertificatePath,
+
+    // exported for testing
+    _getApi: getApi
+};
+
+var acme = require('./cert/acme.js'),
+    apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    caas = require('./cert/caas.js'),
+    cloudron = require('./cloudron.js'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/certificates'),
+    eventlog = require('./eventlog.js'),
+    fs = require('fs'),
+    mailer = require('./mailer.js'),
+    nginx = require('./nginx.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
+    user = require('./user.js'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js'),
+    x509 = require('x509');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function CertificatesError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(CertificatesError, Error);
+CertificatesError.INTERNAL_ERROR = 'Internal Error';
+CertificatesError.INVALID_CERT = 'Invalid certificate';
+CertificatesError.NOT_FOUND = 'Not Found';
+
+function getApi(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getTlsConfig(function (error, tlsConfig) {
+        if (error) return callback(error);
+
+        // use acme if we have altDomain or the tlsConfig is not caas
+        var api = (app.altDomain || tlsConfig.provider) !== 'caas' ? acme : caas;
+
+        var options = { };
+        if (tlsConfig.provider === 'caas') {
+            options.prod = !config.isDev(); // with altDomain, we will choose acme setting based on this
+        } else { // acme
+            options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        }
+
+        // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
+        // we cannot use admin@fqdn because the user might not have set it up.
+        // we simply update the account with the latest email we have each time when getting letsencrypt certs
+        // https://github.com/ietf-wg-acme/acme/issues/30
+        user.getOwner(function (error, owner) {
+            options.email = error ? 'admin@cloudron.io' : owner.email; // can error if not activated yet
+
+            callback(null, api, options);
+        });
+    });
+}
+
+function installAdminCertificate(callback) {
+    settings.getTlsConfig(function (error, tlsConfig) {
+        if (error) return callback(error);
+
+        if (tlsConfig.provider === 'caas') return callback();
+
+        sysinfo.getIp(function (error, ip) {
+            if (error) return callback(error);
+
+            waitForDns(config.adminFqdn(), ip, 'A', { interval: 30000, times: 50000 }, function (error) {
+                if (error) return callback(error);
+
+                ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
+                    if (error) { // currently, this can never happen
+                        debug('Error obtaining certificate. Proceed anyway', error);
+                        return callback();
+                    }
+
+                    nginx.configureAdmin(certFilePath, keyFilePath, callback);
+                });
+            });
+        });
+    });
+}
+
+function isExpiringSync(certFilePath, hours) {
+    assert.strictEqual(typeof certFilePath, 'string');
+    assert.strictEqual(typeof hours, 'number');
+
+    if (!fs.existsSync(certFilePath)) return 2; // not found
+
+    var result = safe.child_process.spawnSync('/usr/bin/openssl', [ 'x509', '-checkend', String(60 * 60 * hours), '-in', certFilePath ]);
+
+    debug('isExpiringSync: %s %s %s', certFilePath, result.stdout.toString('utf8').trim(), result.status);
+
+    return result.status === 1; // 1 - expired 0 - not expired
+}
+
+function renewAll(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('renewAll: Checking certificates for renewal');
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(error);
+
+        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app
+
+        var expiringApps = [ ];
+        for (var i = 0; i < allApps.length; i++) {
+            var appDomain = allApps[i].altDomain || config.appFqdn(allApps[i].location);
+            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
+            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
+
+            if (!safe.fs.existsSync(keyFilePath)) {
+                debug('renewAll: no existing key file for %s. skipping', appDomain);
+                continue;
+            }
+
+            if (isExpiringSync(certFilePath, 24 * 30)) { // expired or not found
+                expiringApps.push(allApps[i]);
+            }
+        }
+
+        debug('renewAll: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
+
+        async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+            var domain = app.altDomain || config.appFqdn(app.location);
+
+            getApi(app, function (error, api, apiOptions) {
+                if (error) return callback(error);
+
+                debug('renewAll: renewing cert for %s with options %j', domain, apiOptions);
+
+                api.getCertificate(domain, apiOptions, function (error) {
+                    var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+                    var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+                    var errorMessage = error ? error.message : '';
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
+                    mailer.certificateRenewed(domain, errorMessage);
+
+                    if (error) {
+                        debug('renewAll: could not renew cert for %s because %s', domain, error);
+
+                        // check if we should fallback if we expire in the coming day
+                        if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
+
+                        debug('renewAll: using fallback certs for %s since it expires soon', domain, error);
+
+                        certFilePath = 'cert/host.cert';
+                        keyFilePath = 'cert/host.key';
+                    } else {
+                        debug('renewAll: certificate for %s renewed', domain);
+                    }
+
+                    // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
+                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                        nginx.configureAdmin.bind(null, certFilePath, keyFilePath)
+                        : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
+
+                    configureFunc(function (ignoredError) {
+                        if (ignoredError) debug('fallbackExpiredCertificates: error reconfiguring app', ignoredError);
+
+                        iteratorCallback(); // move to next app
+                    });
+                });
+            });
+        });
+    });
+}
+
+// note: https://tools.ietf.org/html/rfc4346#section-7.4.2 (certificate_list) requires that the
+// servers certificate appears first (and not the intermediate cert)
+function validateCertificate(cert, key, fqdn) {
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof fqdn, 'string');
+
+    if (cert === null && key === null) return null;
+    if (!cert && key) return new Error('missing cert');
+    if (cert && !key) return new Error('missing key');
+
+    var content;
+    try {
+        content = x509.parseCert(cert);
+    } catch (e) {
+        return new Error('invalid cert: ' + e.message);
+    }
+
+    // check expiration
+    if (content.notAfter < new Date()) return new Error('cert expired');
+
+    function matchesDomain(domain) {
+        if (domain === fqdn) return true;
+        if (domain.indexOf('*') === 0 && domain.slice(2) === fqdn.slice(fqdn.indexOf('.') + 1)) return true;
+
+        return false;
+    }
+
+    // check domain
+    var domains = content.altNames.concat(content.subject.commonName);
+    if (!domains.some(matchesDomain)) return new Error(util.format('cert is not valid for this domain. Expecting %s in %j', fqdn, domains));
+
+    // http://httpd.apache.org/docs/2.0/ssl/ssl_faq.html#verify
+    var certModulus = safe.child_process.execSync('openssl x509 -noout -modulus', { encoding: 'utf8', input: cert });
+    var keyModulus = safe.child_process.execSync('openssl rsa -noout -modulus', { encoding: 'utf8', input: key });
+    if (certModulus !== keyModulus) return new Error('key does not match the cert');
+
+    return null;
+}
+
+function setFallbackCertificate(cert, key, callback) {
+    assert.strictEqual(typeof cert, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateCertificate(cert, key, '*.' + config.fqdn());
+    if (error) return callback(new CertificatesError(CertificatesError.INVALID_CERT, error.message));
+
+    // backup the cert
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.cert'), cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.key'), key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+
+    // copy over fallback cert
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+
+    nginx.reload(function (error) {
+        if (error) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function getFallbackCertificatePath(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // any user fallback cert is always copied over to nginx cert dir
+    callback(null, path.join(paths.NGINX_CERT_DIR, 'host.cert'), path.join(paths.NGINX_CERT_DIR, 'host.key'));
+}
+
+// FIXME: setting admin cert needs to restart the mail container because it uses admin cert
+function setAdminCertificate(cert, key, callback) {
+    assert.strictEqual(typeof cert, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var vhost = config.adminFqdn();
+    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
+
+    var error = validateCertificate(cert, key, vhost);
+    if (error) return callback(new CertificatesError(CertificatesError.INVALID_CERT, error.message));
+
+    // backup the cert
+    if (!safe.fs.writeFileSync(certFilePath, cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(keyFilePath, key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+
+    nginx.configureAdmin(certFilePath, keyFilePath, callback);
+}
+
+function getAdminCertificatePath(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var vhost = config.adminFqdn();
+    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, certFilePath, keyFilePath);
+
+    getFallbackCertificatePath(callback);
+}
+
+function ensureCertificate(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var domain = app.altDomain || config.appFqdn(app.location);
+
+    // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
+    var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+    var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+    if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
+        debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);
+
+        if (!isExpiringSync(userCertFilePath, 24 * 1)) return callback(null, userCertFilePath, userKeyFilePath);
+    }
+
+    debug('ensureCertificate: %s cert require renewal', domain);
+
+    getApi(app, function (error, api, apiOptions) {
+        if (error) return callback(error);
+
+        debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);
+
+        api.getCertificate(domain, apiOptions, function (error, certFilePath, keyFilePath) {
+            if (error) {
+                debug('ensureCertificate: could not get certificate. using fallback certs', error);
+                return callback(null, 'cert/host.cert', 'cert/host.key'); // use fallback certs
+            }
+
+            callback(null, certFilePath, keyFilePath);
+        });
+    });
+}
@@ -5,6 +5,7 @@
 exports = module.exports = {
    get: get,
    getAll: getAll,
+    getAllWithTokenCount: getAllWithTokenCount,
    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
    add: add,
    del: del,
@@ -14,13 +15,7 @@ exports = module.exports = {
    delByAppId: delByAppId,
    delByAppIdAndType: delByAppIdAndType,

-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    _clear: clear
 };

 var assert = require('assert'),
@@ -52,14 +47,24 @@ function getAll(callback) {
    });
 }

+function getAllWithTokenCount(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
 function getAllWithTokenCountByIdentifier(identifier, callback) {
    assert.strictEqual(typeof identifier, 'string');
    assert.strictEqual(typeof callback, 'function');

    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        callback(null, results);

+        callback(null, results);
    });
 }

@@ -71,7 +76,7 @@ function getByAppId(appId, callback) {
        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));

-        return callback(null, result[0]);
+        callback(null, result[0]);
    });
 }

@@ -84,7 +89,7 @@ function getByAppIdAndType(appId, type, callback) {
        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));

-        return callback(null, result[0]);
+        callback(null, result[0]);
    });
 }

@@ -127,7 +132,7 @@ function delByAppId(appId, callback) {
        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));

-        return callback(null);
+        callback(null);
    });
 }

@@ -140,17 +145,17 @@ function delByAppIdAndType(appId, type, callback) {
        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));

-        return callback(null);
+        callback(null);
    });
 }

 function clear(callback) {
    assert.strictEqual(typeof callback, 'function');

-    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.query('DELETE FROM clients WHERE id!="cid-webadmin" AND id!="cid-sdk" AND id!="cid-cli"', function (error) {
        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));

-        return callback(null);
+        callback(null);
    });
 }

@@ -6,9 +6,32 @@ exports = module.exports = {
    add: add,
    get: get,
    del: del,
-    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getAll: getAll,
+    getByAppIdAndType: getByAppIdAndType,
    getClientTokensByUserId: getClientTokensByUserId,
-    delClientTokensByUserId: delClientTokensByUserId
+    delClientTokensByUserId: delClientTokensByUserId,
+    delByAppIdAndType: delByAppIdAndType,
+    addClientTokenByUserId: addClientTokenByUserId,
+    delToken: delToken,
+
+    // keep this in sync with start.sh ADMIN_SCOPES that generates the cid-webadmin
+    SCOPE_APPS: 'apps',
+    SCOPE_DEVELOPER: 'developer',
+    SCOPE_PROFILE: 'profile',
+    SCOPE_CLOUDRON: 'cloudron',
+    SCOPE_SETTINGS: 'settings',
+    SCOPE_USERS: 'users',
+
+    // roles are handled just like the above scopes, they are parallel to scopes
+    // scopes enclose API groups, roles specify the usage role
+    SCOPE_ROLE_SDK: 'roleSdk',
+
+    // client type enums
+    TYPE_EXTERNAL: 'external',
+    TYPE_BUILT_IN: 'built-in',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy'
 };

 var assert = require('assert'),
@@ -16,7 +39,6 @@ var assert = require('assert'),
    hat = require('hat'),
    appdb = require('./appdb.js'),
    tokendb = require('./tokendb.js'),
-    constants = require('./constants.js'),
    async = require('async'),
    clientdb = require('./clientdb.js'),
    DatabaseError = require('./databaseerror.js'),
@@ -43,14 +65,41 @@ function ClientsError(reason, errorOrMessage) {
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
 ClientsError.INVALID_CLIENT = 'Invalid client';
+ClientsError.INVALID_TOKEN = 'Invalid token';
+ClientsError.BAD_FIELD = 'Bad field';
+ClientsError.NOT_FOUND = 'Not found';
+ClientsError.INTERNAL_ERROR = 'Internal Error';
+ClientsError.NOT_ALLOWED = 'Not allowed to remove this client';
+
+function validateName(name) {
+    assert.strictEqual(typeof name, 'string');
+
+    if (name.length < 1) return new ClientsError(ClientsError.BAD_FIELD, 'Name must be atleast 1 character');
+    if (name.length > 128) return new ClientsError(ClientsError.BAD_FIELD, 'Name too long');
+
+    if (/[^a-zA-Z0-9\-]/.test(name)) return new ClientsError(ClientsError.BAD_FIELD, 'Username can only contain alphanumerals and dash');
+
+    return null;
+}

 function validateScope(scope) {
    assert.strictEqual(typeof scope, 'string');

-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
+    var VALID_SCOPES = [
+        exports.SCOPE_APPS,
+        exports.SCOPE_DEVELOPER,
+        exports.SCOPE_PROFILE,
+        exports.SCOPE_CLOUDRON,
+        exports.SCOPE_SETTINGS,
+        exports.SCOPE_USERS,
+        '*',    // includes all scopes, but not roles
+        exports.SCOPE_ROLE_SDK
+    ];

-    // TODO maybe validate all individual scopes if they exist
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE, 'Empty scope not allowed');
+
+    var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE, 'Invalid scope. Available scopes are ' + VALID_SCOPES.join(', '));

    return null;
 }
@@ -62,11 +111,18 @@ function add(appId, type, redirectURI, scope, callback) {
    assert.strictEqual(typeof scope, 'string');
    assert.strictEqual(typeof callback, 'function');

+    // allow whitespace
+    scope = scope.split(',').map(function (s) { return s.trim(); }).join(',');
+
    var error = validateScope(scope);
    if (error) return callback(error);

+    // appId is also client name
+    error = validateName(appId);
+    if (error) return callback(error);
+
    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
+    var clientSecret = hat(8 * 128);

    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
        if (error) return callback(error);
@@ -89,6 +145,7 @@ function get(id, callback) {
    assert.strictEqual(typeof callback, 'function');

    clientdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
        if (error) return callback(error);
        callback(null, result);
    });
@@ -99,24 +156,24 @@ function del(id, callback) {
    assert.strictEqual(typeof callback, 'function');

    clientdb.del(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
        if (error) return callback(error);
        callback(null, result);
    });
 }

-function getAllWithDetailsByUserId(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
+function getAll(callback) {
    assert.strictEqual(typeof callback, 'function');

-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+    clientdb.getAll(function (error, results) {
        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
        if (error) return callback(error);

        var tmp = [];
        async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
-                record.name = constants.ADMIN_NAME;
-                record.location = constants.ADMIN_LOCATION;
+            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
+                // the appId in this case holds the name
+                record.name = record.appId;

                tmp.push(record);

@@ -125,14 +182,13 @@ function getAllWithDetailsByUserId(userId, callback) {

            appdb.get(record.appId, function (error, result) {
                if (error) {
-                    console.error('Failed to get app details for oauth client', result, error);
+                    console.error('Failed to get app details for oauth client', record.appId, error);
                    return callback(null);  // ignore error so we continue listing clients
                }

-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === exports.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';

                record.location = result.location;

@@ -147,15 +203,27 @@ function getAllWithDetailsByUserId(userId, callback) {
    });
 }

+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getByAppIdAndType(appId, type, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
 function getClientTokensByUserId(clientId, userId, callback) {
    assert.strictEqual(typeof clientId, 'string');
    assert.strictEqual(typeof userId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
        if (error && error.reason === DatabaseError.NOT_FOUND) {
            // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                if (error) return callback(error);
                callback(null, []);
            });
@@ -171,10 +239,10 @@ function delClientTokensByUserId(clientId, userId, callback) {
    assert.strictEqual(typeof userId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
        if (error && error.reason === DatabaseError.NOT_FOUND) {
            // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                if (error) return callback(error);
                callback(null);
            });
@@ -184,3 +252,57 @@ function delClientTokensByUserId(clientId, userId, callback) {
        callback(null);
    });
 }
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(appId, type, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null);
+    });
+}
+
+function addClientTokenByUserId(clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        var token = tokendb.generateToken();
+
+        tokendb.add(token, userId, result.id, expiresAt, result.scope, function (error) {
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                accessToken: token,
+                identifier: userId,
+                clientId: result.id,
+                scope: result.id,
+                expires: expiresAt
+            });
+        });
+    });
+}
+
+function delToken(clientId, tokenId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof tokenId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        tokendb.del(tokenId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.INVALID_TOKEN, 'Invalid token'));
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
+    });
+}
@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
@@ -13,37 +11,42 @@ exports = module.exports = {

    sendHeartbeat: sendHeartbeat,

-    update: update,
+    updateToLatest: updateToLatest,
    reboot: reboot,
+    retire: retire,
    migrate: migrate,
-    backup: backup,
-    ensureBackup: ensureBackup,

    isConfiguredSync: isConfiguredSync,

+    checkDiskSpace: checkDiskSpace,
+
    events: new (require('events').EventEmitter)(),

    EVENT_ACTIVATED: 'activated',
-    EVENT_CONFIGURED: 'configured'
+    EVENT_CONFIGURED: 'configured',
+    EVENT_FIRST_RUN: 'firstrun'
 };

 var apps = require('./apps.js'),
-    AppsError = require('./apps.js').AppsError,
    assert = require('assert'),
    async = require('async'),
    backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
-    bytes = require('bytes'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
+    constants = require('./constants.js'),
    debug = require('debug')('box:cloudron'),
+    df = require('node-df'),
+    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    locker = require('./locker.js'),
+    mailer = require('./mailer.js'),
+    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    progress = require('./progress.js'),
    safe = require('safetydance'),
    settings = require('./settings.js'),
+    SettingsError = settings.SettingsError,
    shell = require('./shell.js'),
    subdomains = require('./subdomains.js'),
    superagent = require('superagent'),
@@ -52,37 +55,33 @@ var apps = require('./apps.js'),
    updateChecker = require('./updatechecker.js'),
    user = require('./user.js'),
    UserError = user.UserError,
-    userdb = require('./userdb.js'),
+    user = require('./user.js'),
    util = require('util'),
-    webhooks = require('./webhooks.js');
+    _ = require('underscore');

 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
-    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
+    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
+    RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');

 var NOOP_CALLBACK = function (error) { if (error) debug(error); };

+// result to not depend on the appstore
+const BOX_AND_USER_TEMPLATE = {
+    box: {
+        region: null,
+        size: null,
+        plan: 'Custom Plan'
+    },
+    user: {
+        billing: false,
+        currency: ''
+    }
+};
+
 var gUpdatingDns = false,                // flag for dns update reentrancy
-    gCloudronDetails = null,             // cached cloudron details like region,size...
+    gBoxAndUserDetails = null,         // cached cloudron details like region,size...
    gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet

-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 function CloudronError(reason, errorOrMessage) {
    assert.strictEqual(typeof reason, 'string');
    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -106,25 +105,44 @@ CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
 CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
+CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
+CloudronError.SELF_UPGRADE_NOT_SUPPORTED = 'Self upgrade not supported';

 function initialize(callback) {
    assert.strictEqual(typeof callback, 'function');

-    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    ensureDkimKeySync();

-    syncConfigState(callback);
+    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.on(exports.EVENT_FIRST_RUN, installAppBundle);
+
+    // check activation state for existing cloudrons that do not have first run file
+    // can be removed once cloudrons have been updated
+    isActivated(function (error, activated) {
+        if (error) return callback(error);
+
+        debug('initialize: cloudron %s activated', activated ? '' : 'not');
+
+        if (activated) fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+
+        if (!fs.existsSync(paths.FIRST_RUN_FILE)) {
+            // EE API is sync. do not keep the server waiting
+            debug('initialize: emitting first run event');
+            process.nextTick(function () { exports.events.emit(exports.EVENT_FIRST_RUN); });
+            fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+        }
+
+        syncConfigState(callback);
+    });
 }

 function uninitialize(callback) {
    assert.strictEqual(typeof callback, 'function');

    exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.removeListener(exports.EVENT_FIRST_RUN, installAppBundle);

    callback(null);
 }
@@ -133,6 +151,15 @@ function isConfiguredSync() {
    return gIsConfigured === true;
 }

+function isActivated(callback) {
+    user.getOwner(function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+        if (error) return callback(error);
+
+        callback(null, true);
+    });
+}
+
 function isConfigured(callback) {
    // set of rules to see if we have the configs required for cloudron to function
    // note this checks for missing configs and not invalid configs
@@ -177,54 +204,62 @@ function setTimeZone(ip, callback) {

    debug('setTimeZone ip:%s', ip);

-    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
-        if (error || result.statusCode !== 200) {
-            debug('Failed to get geo location', error);
+    // https://github.com/bluesmoon/node-geoip
+    // https://github.com/runk/node-maxmind
+    // { url: 'http://freegeoip.net/json/%s', jpath: 'time_zone' },
+    // { url: 'http://ip-api.com/json/%s', jpath: 'timezone' },
+    // { url: 'http://geoip.nekudo.com/api/%s', jpath: 'time_zone }
+
+    superagent.get('http://freegeoip.net/json/' + ip).end(function (error, result) {
+        if ((error && !error.response) || result.statusCode !== 200) {
+            debug('Failed to get geo location: %s', error.message);
            return callback(null);
        }

-        if (!result.body.timezone) {
+        if (!result.body.time_zone || typeof result.body.time_zone !== 'string') {
            debug('No timezone in geoip response : %j', result.body);
            return callback(null);
        }

-        debug('Setting timezone to ', result.body.timezone);
+        debug('Setting timezone to ', result.body.time_zone);

-        settings.setTimeZone(result.body.timezone, callback);
+        settings.setTimeZone(result.body.time_zone, callback);
    });
 }

-function activate(username, password, email, ip, callback) {
+function activate(username, password, email, displayName, ip, auditSource, callback) {
    assert.strictEqual(typeof username, 'string');
    assert.strictEqual(typeof password, 'string');
    assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof displayName, 'string');
    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug('activating user:%s email:%s', username, email);

    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region

-    user.createOwner(username, password, email, function (error, userObject) {
+    user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
        if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error.reason === UserError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clients.get('cid-webadmin', function (error, result) {
            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

            // Also generate a token so the admin creation can also act as a login
            var token = tokendb.generateToken();
            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day

-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

                // EE API is sync. do not keep the REST API reponse waiting
                process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });

+                eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
+
                callback(null, { token: token, expires: expires });
            });
        });
@@ -234,7 +269,7 @@ function activate(username, password, email, ip, callback) {
 function getStatus(callback) {
    assert.strictEqual(typeof callback, 'function');

-    userdb.count(function (error, count) {
+    user.count(function (error, count) {
        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

        settings.getCloudronName(function (error, cloudronName) {
@@ -245,47 +280,41 @@ function getStatus(callback) {
                version: config.version(),
                boxVersionsUrl: config.get('boxVersionsUrl'),
                apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool
+                provider: config.provider(),
                cloudronName: cloudronName
            });
        });
    });
 }

-function getCloudronDetails(callback) {
+function getBoxAndUserDetails(callback) {
    assert.strictEqual(typeof callback, 'function');

-    if (gCloudronDetails) return callback(null, gCloudronDetails);
+    if (gBoxAndUserDetails) return callback(null, gBoxAndUserDetails);
+
+    // only supported for caas
+    if (config.provider() !== 'caas') return callback(null, {});

    superagent
        .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
        .query({ token: config.token() })
        .end(function (error, result) {
-            if (error) return callback(error);
-            if (result.status !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, 'Cannot reach appstore'));
+            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));

-            gCloudronDetails = result.body.box;
+            gBoxAndUserDetails = result.body;

-            return callback(null, gCloudronDetails);
+            return callback(null, gBoxAndUserDetails);
        });
 }

 function getConfig(callback) {
    assert.strictEqual(typeof callback, 'function');

-    getCloudronDetails(function (error, result) {
-        if (error) {
-            debug('Failed to fetch cloudron details.', error);
+    getBoxAndUserDetails(function (error, result) {
+        if (error) debug('Failed to fetch cloudron details.', error.reason, error.message);

-            // set fallback values to avoid dependency on appstore
-            result = {
-                region: result ? result.region : null,
-                size: result ? result.size : null
-            };
-        }
-
-        // We rely at the moment on the size being specified in 512mb,1gb,...
-        // TODO provide that number from the appstore
-        var memory = bytes(result.size) || 0;
+        result = _.extend(BOX_AND_USER_TEMPLATE, result || {});

        settings.getCloudronName(function (error, cloudronName) {
            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
@@ -293,21 +322,29 @@ function getConfig(callback) {
            settings.getDeveloperMode(function (error, developerMode) {
                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

-                callback(null, {
-                    apiServerOrigin: config.apiServerOrigin(),
-                    webServerOrigin: config.webServerOrigin(),
-                    isDev: config.isDev(),
-                    fqdn: config.fqdn(),
-                    ip: sysinfo.getIp(),
-                    version: config.version(),
-                    update: updateChecker.getUpdateInfo(),
-                    progress: progress.get(),
-                    isCustomDomain: config.isCustomDomain(),
-                    developerMode: developerMode,
-                    region: result.region,
-                    size: result.size,
-                    memory: memory,
-                    cloudronName: cloudronName
+                sysinfo.getIp(function (error, ip) {
+                    if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+                    callback(null, {
+                        apiServerOrigin: config.apiServerOrigin(),
+                        webServerOrigin: config.webServerOrigin(),
+                        isDev: config.isDev(),
+                        fqdn: config.fqdn(),
+                        ip: ip,
+                        version: config.version(),
+                        update: updateChecker.getUpdateInfo(),
+                        progress: progress.get(),
+                        isCustomDomain: config.isCustomDomain(),
+                        developerMode: developerMode,
+                        region: result.box.region,
+                        size: result.box.size,
+                        billing: !!result.user.billing,
+                        plan: result.box.plan,
+                        currency: result.user.currency,
+                        memory: os.totalmem(),
+                        provider: config.provider(),
+                        cloudronName: cloudronName
+                    });
                });
            });
        });
@@ -315,15 +352,31 @@ function getConfig(callback) {
 }

 function sendHeartbeat() {
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
+    if (!config.token()) return;

+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
-        if (error) debug('Error sending heartbeat.', error);
+        if (error && !error.response) debug('Network error sending heartbeat.', error);
        else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
        else debug('Heartbeat sent to %s', url);
    });
 }

+function ensureDkimKeySync() {
+    var dkimPrivateKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/private');
+    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
+
+    if (fs.existsSync(dkimPrivateKeyFile) && fs.existsSync(dkimPublicKeyFile)) {
+        debug('DKIM keys already present');
+        return;
+    }
+
+    debug('Generating new DKIM keys');
+
+    safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024');
+    safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM');
+}
+
 function readDkimPublicKeySync() {
    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
    var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
@@ -339,6 +392,7 @@ function readDkimPublicKeySync() {
    return publicKey;
 }

+// NOTE: if you change the SPF record here, be sure the wait check in mailer.js
 function txtRecordsWithSpf(callback) {
    assert.strictEqual(typeof callback, 'function');

@@ -352,16 +406,16 @@ function txtRecordsWithSpf(callback) {
        for (i = 0; i < txtRecords.length; i++) {
            if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF

-            validSpf = txtRecords[i].indexOf(' a:' + config.fqdn() + ' ') !== -1;
+            validSpf = txtRecords[i].indexOf(' a:' + config.adminFqdn() + ' ') !== -1;
            break;
        }

        if (validSpf) return callback(null, null);

        if (i == txtRecords.length) {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ~all"';
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ~all"';
        } else {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
        }

        return callback(null, txtRecords);
@@ -383,51 +437,61 @@ function addDnsRecords() {
    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';

    var dkimKey = readDkimPublicKeySync();
-    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
+    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));

-    var nakedDomainRecord = { subdomain: '', type: 'A', values: [ sysinfo.getIp() ] };
-    var webadminRecord = { subdomain: 'my', type: 'A', values: [ sysinfo.getIp() ] };
-    // t=s limits the domainkey to this domain and not it's subdomains
-    var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
-    // DMARC requires special setup if report email id is in different domain
-    var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
+    sysinfo.getIp(function (error, ip) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

-    var records = [ ];
-    if (config.isCustomDomain()) {
-        records.push(webadminRecord);
-        records.push(dkimRecord);
-    } else {
-        records.push(nakedDomainRecord);
-        records.push(webadminRecord);
-        records.push(dkimRecord);
-        records.push(dmarcRecord);
-    }
+        var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
+        // t=s limits the domainkey to this domain and not it's subdomains
+        var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
+        // DMARC requires special setup if report email id is in different domain
+        var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };

-    debug('addDnsRecords: %j', records);
+        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };

-    async.retry({ times: 10, interval: 20000 }, function (retryCallback) {
-        txtRecordsWithSpf(function (error, txtRecords) {
-            if (error) return retryCallback(error);
+        var records = [ ];
+        if (config.isCustomDomain()) {
+            records.push(webadminRecord);
+            records.push(dkimRecord);
+            records.push(mxRecord);
+        } else {
+            // for non-custom domains, we show a nakeddomain.html page
+            var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };

-            if (txtRecords) records.push({ subdomain: '', type: 'TXT', values: txtRecords });
+            records.push(nakedDomainRecord);
+            records.push(webadminRecord);
+            records.push(dkimRecord);
+            records.push(dmarcRecord);
+            records.push(mxRecord);
+        }

-            debug('addDnsRecords: will update %j', records);
+        debug('addDnsRecords: %j', records);

-            async.mapSeries(records, function (record, iteratorCallback) {
-                subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
-            }, function (error, changeIds) {
-                if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
-                else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
+        async.retry({ times: 10, interval: 20000 }, function (retryCallback) {
+            txtRecordsWithSpf(function (error, txtRecords) {
+                if (error) return retryCallback(error);

-                retryCallback(error);
+                if (txtRecords) records.push({ subdomain: '', type: 'TXT', values: txtRecords });
+
+                debug('addDnsRecords: will update %j', records);
+
+                async.mapSeries(records, function (record, iteratorCallback) {
+                    subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
+                }, function (error, changeIds) {
+                    if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
+                    else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
+
+                    retryCallback(error);
+                });
            });
+        }, function (error) {
+            gUpdatingDns = false;
+
+            debug('addDnsRecords: done updating records with error:', error);
+
+            callback(error);
        });
-    }, function (error) {
-        gUpdatingDns = false;
-
-        debug('addDnsRecords: done updating records with error:', error);
-
-        callback(error);
    });
 }

@@ -435,51 +499,9 @@ function reboot(callback) {
    shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }

-function migrate(size, region, callback) {
-    assert.strictEqual(typeof size, 'string');
-    assert.strictEqual(typeof region, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_MIGRATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    function unlock(error) {
-        if (error) {
-            debug('Failed to migrate', error);
-            locker.unlock(locker.OP_MIGRATE);
-        } else {
-            debug('Migration initiated successfully');
-            // do not unlock; cloudron is migrating
-        }
-
-        return;
-    }
-
-    // initiate the migration in the background
-    backupBoxAndApps(function (error, restoreKey) {
-        if (error) return unlock(error);
-
-        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
-
-        superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send({ size: size, region: region, restoreKey: restoreKey })
-          .end(function (error, result) {
-            if (error) return unlock(error);
-            if (result.status === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.status === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.status !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
-
-            return unlock(null);
-        });
-    });
-
-    callback(null);
-}
-
-function update(boxUpdateInfo, callback) {
+function update(boxUpdateInfo, auditSource, callback) {
    assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    if (!boxUpdateInfo) return callback(null);
@@ -487,8 +509,18 @@ function update(boxUpdateInfo, callback) {
    var error = locker.lock(locker.OP_BOX_UPDATE);
    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));

+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
+    // ensure tools can 'wait' on progress
+    progress.set(progress.UPDATE, 0, 'Starting');
+
    // initiate the update/upgrade but do not wait for it
-    if (boxUpdateInfo.upgrade) {
+    if (config.version().match(/[-+]/) !== null && config.version().replace(/[-+].*/, '') === boxUpdateInfo.version) {
+        doShortCircuitUpdate(boxUpdateInfo, function (error) {
+            if (error) debug('Short-circuit update failed', error);
+            locker.unlock(locker.OP_BOX_UPDATE);
+        });
+    } else if (boxUpdateInfo.upgrade) {
        debug('Starting upgrade');
        doUpgrade(boxUpdateInfo, function (error) {
            if (error) {
@@ -509,6 +541,29 @@ function update(boxUpdateInfo, callback) {
    callback(null);
 }

+
+function updateToLatest(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var boxUpdateInfo = updateChecker.getUpdateInfo().box;
+    if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
+
+    if (boxUpdateInfo.upgrade && config.provider() !== 'caas') return callback(new CloudronError(CloudronError.SELF_UPGRADE_NOT_SUPPORTED));
+
+    update(boxUpdateInfo, auditSource, callback);
+}
+
+function doShortCircuitUpdate(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
+
+    debug('Starting short-circuit from prerelease version %s to release version %s', config.version(), boxUpdateInfo.version);
+    config.setVersion(boxUpdateInfo.version);
+    progress.clear(progress.UPDATE);
+    updateChecker.resetUpdateInfo();
+    callback();
+}
+
 function doUpgrade(boxUpdateInfo, callback) {
    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');

@@ -519,21 +574,21 @@ function doUpgrade(boxUpdateInfo, callback) {

    progress.set(progress.UPDATE, 5, 'Backing up for upgrade');

-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'upgrader' }, function (error) {
        if (error) return upgradeError(error);

        superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
          .query({ token: config.token() })
          .send({ version: boxUpdateInfo.version })
          .end(function (error, result) {
-            if (error) return upgradeError(new Error('Error making upgrade request: ' + error));
-            if (result.status !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
+            if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
+            if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));

            progress.set(progress.UPDATE, 10, 'Updating base system');

            // no need to unlock since this is the last thing we ever do on this box
-
-            callback(null);
+            callback();
+            retire('upgrade');
        });
    });
 }
@@ -548,179 +603,174 @@ function doUpdate(boxUpdateInfo, callback) {

    progress.set(progress.UPDATE, 5, 'Backing up for update');

-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
        if (error) return updateError(error);

-        // fetch a signed sourceTarballUrl
-        superagent.get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/sourcetarballurl')
-          .query({ token: config.token(), boxVersion: boxUpdateInfo.version })
-          .end(function (error, result) {
-            if (error) return updateError(new Error('Error fetching sourceTarballUrl: ' + error));
-            if (result.status !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.status));
-            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + JSON.stringify(result.body)));
+        // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
+        var args = {
+            sourceTarballUrl: boxUpdateInfo.sourceTarballUrl,

-            // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
-            var args = {
-                sourceTarballUrl: result.body.url,
+            // this data is opaque to the installer
+            data: {
+                provider: config.provider(),
+                token: config.token(),
+                apiServerOrigin: config.apiServerOrigin(),
+                webServerOrigin: config.webServerOrigin(),
+                fqdn: config.fqdn(),
+                tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                isCustomDomain: config.isCustomDomain(),

-                // this data is opaque to the installer
-                data: {
+                appstore: {
+                    token: config.token(),
+                    apiServerOrigin: config.apiServerOrigin()
+                },
+                caas: {
                    token: config.token(),
                    apiServerOrigin: config.apiServerOrigin(),
-                    webServerOrigin: config.webServerOrigin(),
-                    fqdn: config.fqdn(),
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    isCustomDomain: config.isCustomDomain(),
+                    webServerOrigin: config.webServerOrigin()
+                },

-                    appstore: {
-                        token: config.token(),
-                        apiServerOrigin: config.apiServerOrigin()
-                    },
-                    caas: {
-                        token: config.token(),
-                        apiServerOrigin: config.apiServerOrigin(),
-                        webServerOrigin: config.webServerOrigin()
-                    },
-                    tlsConfig: {
-                        provider: 'caas',
-                        cert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
-                        key: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    },
+                version: boxUpdateInfo.version,
+                boxVersionsUrl: config.get('boxVersionsUrl')
+            }
+        };

-                    version: boxUpdateInfo.version,
-                    boxVersionsUrl: config.get('boxVersionsUrl')
-                }
-            };
+        debug('updating box %j', args);

-            debug('updating box %j', args);
+        superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
+            if (error && !error.response) return updateError(error);
+            if (result.statusCode !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));

-            superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
-                if (error) return updateError(error);
-                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));
+            progress.set(progress.UPDATE, 10, 'Updating cloudron software');

-                progress.set(progress.UPDATE, 10, 'Updating cloudron software');
-
-                callback(null);
-            });
+            callback(null);
        });

        // Do not add any code here. The installer script will stop the box code any instant
    });
 }

-function backup(callback) {
+function installAppBundle(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    var bundle = config.get('appBundle');
+
+    if (!bundle || bundle.length === 0) {
+        debug('installAppBundle: no bundle set');
+        return callback();
+    }
+
+    async.eachSeries(bundle, function (appInfo, iteratorCallback) {
+        debug('autoInstall: installing %s at %s', appInfo.appstoreId, appInfo.location);
+
+        var data = {
+            appStoreId: appInfo.appstoreId,
+            location: appInfo.location,
+            portBindings: appInfo.portBindings || null,
+            accessRestriction: appInfo.accessRestriction || null,
+        };
+
+        apps.install(data, { userId: null, username: 'autoinstaller' }, iteratorCallback);
+    }, function (error) {
+        if (error) debug('autoInstallApps: ', error);
+
+        callback();
+    });
+}
+
+function checkDiskSpace(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Checking disk space');
+
+    df(function (error, entries) {
+        if (error) {
+            debug('df error %s', error.message);
+            mailer.outOfDiskSpace(error.message);
+            return callback();
+        }
+
+        var oos = entries.some(function (entry) {
+            return (entry.mount === paths.DATA_DIR && entry.capacity >= 0.90) ||
+                   (entry.mount === '/' && entry.used <= (1.25 * 1024 * 1024)); // 1.5G
+        });
+
+        debug('Disk space checked. ok: %s', !oos);
+
+        if (oos) mailer.outOfDiskSpace(JSON.stringify(entries, null, 4));
+
+        callback();
+    });
+}
+
+function retire(reason, info, callback) {
+    assert(reason === 'migrate' || reason === 'upgrade');
+    info = info || { };
+    callback = callback || NOOP_CALLBACK;
+
+    var data = {
+        apiServerOrigin: config.apiServerOrigin(),
+        isCustomDomain: config.isCustomDomain(),
+        fqdn: config.fqdn()
+    };
+    shell.sudo('retire', [ RETIRE_CMD, reason, JSON.stringify(info), JSON.stringify(data) ], callback);
+}
+
+function doMigrate(options, callback) {
+    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var error = locker.lock(locker.OP_FULL_BACKUP);
+    var error = locker.lock(locker.OP_MIGRATE);
    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));

-    // clearing backup ensures tools can 'wait' on progress
-    progress.clear(progress.BACKUP);
+    function unlock(error) {
+        debug('Failed to migrate', error);
+        locker.unlock(locker.OP_MIGRATE);
+        progress.set(progress.MIGRATE, -1, 'Backup failed: ' + error.message);
+    }

-    // start the backup operation in the background
-    backupBoxAndApps(function (error) {
-        if (error) console.error('backup failed.', error);
+    progress.set(progress.MIGRATE, 10, 'Backing up for migration');

-        locker.unlock(locker.OP_FULL_BACKUP);
+    // initiate the migration in the background
+    backups.backupBoxAndApps({ userId: null, username: 'migrator' }, function (error, backupId) {
+        if (error) return unlock(error);
+
+        debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region);
+
+        options.restoreKey = backupId;
+
+        superagent
+          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+          .query({ token: config.token() })
+          .send(options)
+          .end(function (error, result) {
+            if (error && !error.response) return unlock(error); // network error
+            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            progress.set(progress.MIGRATE, 10, 'Migrating');
+
+            retire('migrate', _.pick(options, 'domain', 'size', 'region'));
+        });
    });

    callback(null);
 }

-function ensureBackup(callback) {
-    callback = callback || function () { };
+function migrate(options, callback) {
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');

-    backups.getAllPaged(1, 1, function (error, backups) {
-        if (error) {
-            debug('Unable to list backups', error);
-            return callback(error); // no point trying to backup if appstore is down
-        }
+    if (!options.domain) return doMigrate(options, callback);

-        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
-            debug('Previous backup was %j, no need to backup now', backups[0]);
-            return callback(null);
-        }
+    var dnsConfig = _.pick(options, 'domain', 'provider', 'accessKeyId', 'secretAccessKey', 'region', 'endpoint');

-        backup(callback);
-    });
-}
-
-function backupBoxWithAppBackupIds(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
-
-    backups.getBackupUrl(null /* app */, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
+    settings.setDnsConfig(dnsConfig, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

-        debug('backup: url %s', result.url);
-
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            debug('backup: successful');
-
-            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                if (error) return callback(error);
-                callback(null, result.id);
-            });
-        });
-    });
-}
-
-// this function expects you to have a lock
-function backupBox(callback) {
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
-
-        backupBoxWithAppBackupIds(appBackupIds, callback);
-    });
-}
-
-// this function expects you to have a lock
-function backupBoxAndApps(callback) {
-    callback = callback || function () { }; // callback can be empty for timer triggered backup
-
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var processed = 0;
-        var step = 100/(allApps.length+1);
-
-        progress.set(progress.BACKUP, processed, '');
-
-        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
-            ++processed;
-
-            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                if (error && error.reason !== AppsError.BAD_STATE) {
-                    debugApp(app, 'Unable to backup', error);
-                    return iteratorCallback(error);
-                }
-
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
-                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
-            });
-        }, function appsBackedUp(error, backupIds) {
-            if (error) {
-                progress.set(progress.BACKUP, 100, error.message);
-                return callback(error);
-            }
-
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
-
-            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
-                progress.set(progress.BACKUP, 100, error ? error.message : '');
-                callback(error, restoreKey);
-            });
-        });
+        doMigrate(options, callback);
    });
 }
@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.stat">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
    Instance "<%= appId %>-memory"
    Separator " \\n"
    <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
    </Result>
  </Table>

-  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
    Instance "<%= appId %>-memory"
    Separator "\\n"
    <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
    </Result>
  </Table>

-  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker-<%= containerId %>.scope/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
    Instance "<%= appId %>-cpu"
    Separator " \\n"
    <Result>
@@ -1,11 +1,7 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
    baseDir: baseDir,
-    dnsInSync: dnsInSync,
-    setDnsInSync: setDnsInSync,

    // values set here will be lost after a upgrade/update. use the sqlite database
    // for persistent values that need to be backed up
@@ -17,24 +13,29 @@ exports = module.exports = {
    TEST: process.env.BOX_ENV === 'test',

    // convenience getters
+    provider: provider,
    apiServerOrigin: apiServerOrigin,
    webServerOrigin: webServerOrigin,
    fqdn: fqdn,
    token: token,
    version: version,
+    setVersion: setVersion,
    isCustomDomain: isCustomDomain,
    database: database,

    // these values are derived
    adminOrigin: adminOrigin,
    internalAdminOrigin: internalAdminOrigin,
+    sysadminOrigin: sysadminOrigin, // caas routes
+    adminFqdn: adminFqdn,
+    mailFqdn: mailFqdn,
    appFqdn: appFqdn,
    zoneName: zoneName,

    isDev: isDev,

    // for testing resets to defaults
-    _reset: initConfig
+    _reset: _reset
 };

 var assert = require('assert'),
@@ -55,32 +56,34 @@ function baseDir() {

 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');

-function dnsInSync() {
-    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
-}
-
-function setDnsInSync(content) {
-    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
-}
-
 function saveSync() {
    fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }

+function _reset (callback) {
+    safe.fs.unlinkSync(cloudronConfigFileName);
+
+    initConfig();
+
+    if (callback) callback();
+}
+
 function initConfig() {
    // setup defaults
    data.fqdn = 'localhost';

    data.token = null;
-    data.adminEmail = null;
    data.boxVersionsUrl = null;
    data.version = null;
    data.isCustomDomain = false;
    data.webServerOrigin = null;
-    data.internalPort = 3001;
+    data.smtpPort = 2525; // // this value comes from mail container
+    data.sysadminPort = 3001;
    data.ldapPort = 3002;
    data.oauthProxyPort = 3003;
    data.simpleAuthPort = 3004;
+    data.provider = 'caas';
+    data.appBundle = [ ];

    if (exports.CLOUDRON) {
        data.port = 3000;
@@ -155,6 +158,14 @@ function appFqdn(location) {
    return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
 }

+function adminFqdn() {
+    return appFqdn(constants.ADMIN_LOCATION);
+}
+
+function mailFqdn() {
+    return appFqdn(constants.MAIL_LOCATION);
+}
+
 function adminOrigin() {
    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
@@ -163,6 +174,10 @@ function internalAdminOrigin() {
    return 'http://127.0.0.1:' + get('port');
 }

+function sysadminOrigin() {
+    return 'http://127.0.0.1:' + get('sysadminPort');
+}
+
 function token() {
    return get('token');
 }
@@ -171,6 +186,10 @@ function version() {
    return get('version');
 }

+function setVersion(version) {
+    set('version', version);
+}
+
 function isCustomDomain() {
    return get('isCustomDomain');
 }
@@ -189,3 +208,7 @@ function database() {
 function isDev() {
    return /dev/i.test(get('boxVersionsUrl'));
 }
+
+function provider() {
+    return get('provider');
+}
@@ -4,9 +4,18 @@
 exports = module.exports = {
    ADMIN_LOCATION: 'my',
    API_LOCATION: 'api', // this is unused but reserved for future use (#403)
+    SMTP_LOCATION: 'smtp',
+    IMAP_LOCATION: 'imap',
+    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
+    POSTMAN_LOCATION: 'postman', // used in dovecot bounces
+
    ADMIN_NAME: 'Settings',

    ADMIN_CLIENT_ID: 'webadmin', // oauth client id
-    ADMIN_APPID: 'admin' // admin appid (settingsdb)
+    ADMIN_APPID: 'admin', // admin appid (settingsdb)
+
+    GHOST_USER_FILE: '/tmp/cloudron_ghost.json',
+
+    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
 };

@@ -7,10 +7,13 @@ exports = module.exports = {

 var apps = require('./apps.js'),
    assert = require('assert'),
+    backups = require('./backups.js'),
+    certificates = require('./certificates.js'),
    cloudron = require('./cloudron.js'),
    config = require('./config.js'),
    CronJob = require('cron').CronJob,
    debug = require('debug')('box:cron'),
+    eventlog = require('./eventlog.js'),
    janitor = require('./janitor.js'),
    scheduler = require('./scheduler.js'),
    settings = require('./settings.js'),
@@ -23,9 +26,13 @@ var gAutoupdaterJob = null,
    gBackupJob = null,
    gCleanupTokensJob = null,
    gDockerVolumeCleanerJob = null,
-    gSchedulerSyncJob = null;
+    gSchedulerSyncJob = null,
+    gCertificateRenewJob = null,
+    gCheckDiskSpaceJob = null,
+    gCleanupEventlogJob = null;

 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var AUDIT_SOURCE = { userId: null, username: 'cron' };

 // cron format
 // Seconds: 0-59
@@ -61,8 +68,16 @@ function recreateJobs(unusedTimeZone, callback) {

        if (gBackupJob) gBackupJob.stop();
        gBackupJob = new CronJob({
-            cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: cloudron.ensureBackup,
+            cronTime: '00 00 */4 * * *', // every 4 hours. backups.ensureBackup() will only trigger a backup once per day
+            onTick: backups.ensureBackup.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gCheckDiskSpaceJob) gCheckDiskSpaceJob.stop();
+        gCheckDiskSpaceJob = new CronJob({
+            cronTime: '00 30 */4 * * *', // every 4 hours
+            onTick: cloudron.checkDiskSpace,
            start: true,
            timeZone: allSettings[settings.TIME_ZONE_KEY]
        });
@@ -91,6 +106,14 @@ function recreateJobs(unusedTimeZone, callback) {
            timeZone: allSettings[settings.TIME_ZONE_KEY]
        });

+        if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
+        gCleanupEventlogJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: eventlog.cleanup,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
        if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
        gDockerVolumeCleanerJob = new CronJob({
            cronTime: '00 00 */12 * * *', // every 12 hours
@@ -107,6 +130,14 @@ function recreateJobs(unusedTimeZone, callback) {
            timeZone: allSettings[settings.TIME_ZONE_KEY]
        });

+        if (gCertificateRenewJob) gCertificateRenewJob.stop();
+        gCertificateRenewJob = new CronJob({
+            cronTime: '00 00 */12 * * *', // every 12 hours
+            onTick: certificates.renewAll.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
        settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
        settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY]);
@@ -134,10 +165,10 @@ function autoupdatePatternChanged(pattern) {
            var updateInfo = updateChecker.getUpdateInfo();
            if (updateInfo.box) {
                debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
            } else if (updateInfo.apps) {
                debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+                apps.updateApps(updateInfo.apps, AUDIT_SOURCE, NOOP_CALLBACK);
            } else {
                debug('No auto updates available');
            }
@@ -173,11 +204,17 @@ function uninitialize(callback) {
    if (gCleanupTokensJob) gCleanupTokensJob.stop();
    gCleanupTokensJob = null;

+    if (gCleanupEventlogJob) gCleanupEventlogJob.stop();
+    gCleanupEventlogJob = null;
+
    if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
    gDockerVolumeCleanerJob = null;

    if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
    gSchedulerSyncJob = null;

+    if (gCertificateRenewJob) gCertificateRenewJob.stop();
+    gCertificateRenewJob = null;
+
    callback();
 }
@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
@@ -116,10 +114,14 @@ function clear(callback) {
    async.series([
        require('./appdb.js')._clear,
        require('./authcodedb.js')._clear,
+        require('./backupdb.js')._clear,
        require('./clientdb.js')._clear,
        require('./tokendb.js')._clear,
+        require('./groupdb.js')._clear,
        require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear
+        require('./settingsdb.js')._clear,
+        require('./eventlogdb.js')._clear,
+        require('./mailboxdb.js')._clear
    ], callback);
 }

@@ -30,3 +30,4 @@ DatabaseError.INTERNAL_ERROR = 'Internal error';
 DatabaseError.ALREADY_EXISTS = 'Entry already exist';
 DatabaseError.NOT_FOUND = 'Record not found';
 DatabaseError.BAD_FIELD = 'Invalid field';
+DatabaseError.IN_USE = 'In Use';
@@ -5,7 +5,7 @@
 exports = module.exports = {
    DeveloperError: DeveloperError,

-    enabled: enabled,
+    isEnabled: isEnabled,
    setEnabled: setEnabled,
    issueDeveloperToken: issueDeveloperToken,
    getNonApprovedApps: getNonApprovedApps
@@ -13,6 +13,9 @@ exports = module.exports = {

 var assert = require('assert'),
    config = require('./config.js'),
+    clients = require('./clients.js'),
+    debug = require('debug')('box:developer'),
+    eventlog = require('./eventlog.js'),
    tokendb = require('./tokendb.js'),
    settings = require('./settings.js'),
    superagent = require('superagent'),
@@ -38,8 +41,9 @@ function DeveloperError(reason, errorOrMessage) {
 }
 util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
+DeveloperError.EXTERNAL_ERROR = 'External Error';

-function enabled(callback) {
+function isEnabled(callback) {
    assert.strictEqual(typeof callback, 'function');

    settings.getDeveloperMode(function (error, enabled) {
@@ -48,27 +52,35 @@ function enabled(callback) {
    });
 }

-function setEnabled(enabled, callback) {
+function setEnabled(enabled, auditSource, callback) {
    assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    settings.setDeveloperMode(enabled, function (error) {
        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        eventlog.add(eventlog.ACTION_CLI_MODE, auditSource, { enabled: enabled });
+
        callback(null);
    });
 }

-function issueDeveloperToken(user, callback) {
+function issueDeveloperToken(user, auditSource, callback) {
    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    var token = tokendb.generateToken();
    var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var scopes = '*,' + clients.SCOPE_ROLE_SDK;

-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
+    tokendb.add(token, user.id, 'cid-cli', expiresAt, scopes, function (error) {
        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));

-        callback(null, { token: token, expiresAt: expiresAt });
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
+
+        callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
    });
 }

@@ -77,8 +89,12 @@ function getNonApprovedApps(callback) {

    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
-        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
-        if (result.status !== 200) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+        if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
+        if (result.statusCode === 401 || result.statusCode === 403) {
+            debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);
+            return callback(null, []);
+        }
+        if (result.statusCode !== 200) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));

        callback(null, result.body.apps || []);
    });
@@ -40,9 +40,9 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
        .query({ token: dnsConfig.token })
        .send(data)
        .end(function (error, result) {
-            if (error) return callback(error);
-            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
-            if (result.status !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(error);
+            if (result.statusCode === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));

            return callback(null, result.body.changeId);
        });
@@ -63,8 +63,8 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {
        .get(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
        .query({ token: dnsConfig.token, type: type })
        .end(function (error, result) {
-            if (error) return callback(error);
-            if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(error);
+            if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));

            return callback(null, result.body.values);
        });
@@ -107,10 +107,10 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
        .query({ token: dnsConfig.token })
        .send(data)
        .end(function (error, result) {
-            if (error) return callback(error);
-            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
-            if (result.status === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND));
-            if (result.status !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(error);
+            if (result.statusCode === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND));
+            if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));

            return callback(null);
        });
@@ -127,8 +127,8 @@ function getChangeStatus(dnsConfig, changeId, callback) {
        .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
        .query({ token: dnsConfig.token })
        .end(function (error, result) {
-            if (error) return callback(error);
-            if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (error && !error.response) return callback(error);
+            if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));

            return callback(null, result.body.status);
        });
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -7,12 +5,14 @@ exports = module.exports = {
    get: get,
    del: del,
    update: update,
-    getChangeStatus: getChangeStatus
+    getChangeStatus: getChangeStatus,
+
+    // not part of "dns" interface
+    getHostedZone: getHostedZone
 };

 var assert = require('assert'),
    AWS = require('aws-sdk'),
-    config = require('../config.js'),
    debug = require('debug')('box:dns/route53'),
    SubdomainError = require('../subdomains.js').SubdomainError,
    util = require('util'),
@@ -39,7 +39,8 @@ function getZoneByName(dnsConfig, zoneName, callback) {

    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
    route53.listHostedZones({}, function (error, result) {
-        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));

        var zone = result.HostedZones.filter(function (zone) {
            return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
@@ -51,6 +52,24 @@ function getZoneByName(dnsConfig, zoneName, callback) {
    });
 }

+function getHostedZone(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+ 
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.getHostedZone({ Id: zone.Id }, function (error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+
+            callback(null, result);
+        });
+    });
+}
+
 function add(dnsConfig, zoneName, subdomain, type, values, callback) {
    assert.strictEqual(typeof dnsConfig, 'object');
    assert.strictEqual(typeof zoneName, 'string');
@@ -64,7 +83,7 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
    getZoneByName(dnsConfig, zoneName, function (error, zone) {
        if (error) return callback(error);

-        var fqdn = config.appFqdn(subdomain);
+        var fqdn = subdomain === '' ? zoneName : subdomain + '.' + zoneName;
        var records = values.map(function (v) { return { Value: v }; });

        var params = {
@@ -84,11 +103,9 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {

        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
        route53.changeResourceRecordSets(params, function(error, result) {
-            if (error && error.code === 'PriorRequestNotComplete') {
-                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
-            } else if (error) {
-                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
-            }
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error && error.code === 'PriorRequestNotComplete') return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));

            callback(null, result.ChangeInfo.Id);
        });
@@ -131,7 +148,8 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {

        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
        route53.listResourceRecordSets(params, function (error, result) {
-            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
            if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
            if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);

@@ -153,7 +171,7 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
    getZoneByName(dnsConfig, zoneName, function (error, zone) {
        if (error) return callback(error);

-        var fqdn = config.appFqdn(subdomain);
+        var fqdn = subdomain === '' ? zoneName : subdomain + '.' + zoneName;
        var records = values.map(function (v) { return { Value: v }; });

        var resourceRecordSet = {
@@ -175,21 +193,22 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {

        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
            if (error && error.message && error.message.indexOf('it was not found') !== -1) {
-                debug('delSubdomain: resource record set not found.', error);
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                debug('del: resource record set not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
            } else if (error && error.code === 'NoSuchHostedZone') {
-                debug('delSubdomain: hosted zone not found.', error);
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                debug('del: hosted zone not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
            } else if (error && error.code === 'PriorRequestNotComplete') {
-                debug('delSubdomain: resource is still busy', error);
-                return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                debug('del: resource is still busy', error);
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
            } else if (error && error.code === 'InvalidChangeBatch') {
-                debug('delSubdomain: invalid change batch. No such record to be deleted.');
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                debug('del: invalid change batch. No such record to be deleted.');
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
            } else if (error) {
-                debug('delSubdomain: error', error);
-                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                debug('del: error', error);
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
            }

            callback(null);
@@ -206,6 +225,7 @@ function getChangeStatus(dnsConfig, changeId, callback) {

    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
    route53.getChange({ Id: changeId }, function (error, result) {
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
        if (error) return callback(error);

        callback(null, result.ChangeInfo.Status);
@@ -1,30 +1,24 @@
 'use strict';

-var addons = require('./addons.js'),
-    async = require('async'),
-    assert = require('assert'),
-    config = require('./config.js'),
-    debug = require('debug')('box:src/docker.js'),
-    Docker = require('dockerode'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    util = require('util'),
-    _ = require('underscore');
-
 exports = module.exports = {
    connection: connectionInstance(),
    downloadImage: downloadImage,
    createContainer: createContainer,
    startContainer: startContainer,
    stopContainer: stopContainer,
+    stopContainerByName: stopContainer,
    stopContainers: stopContainers,
    deleteContainer: deleteContainer,
+    deleteContainerByName: deleteContainer,
    deleteImage: deleteImage,
    deleteContainers: deleteContainers,
-    createSubcontainer: createSubcontainer
+    createSubcontainer: createSubcontainer,
+    getContainerIdByIp: getContainerIdByIp,
+    execContainer: execContainer
 };

 function connectionInstance() {
+    var Docker = require('dockerode');
    var docker;

    if (process.env.BOX_ENV === 'test') {
@@ -40,6 +34,19 @@ function connectionInstance() {
    return docker;
 }

+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    child_process = require('child_process'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/docker.js'),
+    once = require('once'),
+    safe = require('safetydance'),
+    spawn = child_process.spawn,
+    util = require('util'),
+    _ = require('underscore');
+
 function debugApp(app, args) {
    assert(!app || typeof app === 'object');

@@ -52,7 +59,7 @@ function targetBoxVersion(manifest) {

    if ('minBoxVersion' in manifest) return manifest.minBoxVersion;

-    return '0.0.1';
+    return '99999.99999.99999'; // compatible with the latest version
 }

 function pullImage(manifest, callback) {
@@ -125,16 +132,18 @@ function createSubcontainer(app, name, cmd, options, callback) {
    assert.strictEqual(typeof callback, 'function');

    var docker = exports.connection,
-        isAppContainer = !cmd;
+        isAppContainer = !cmd; // non app-containers are like scheduler containers

    var manifest = app.manifest;
+    var developmentMode = !!manifest.developmentMode;
    var exposedPorts = {}, dockerPortBindings = { };
+    var domain = app.altDomain || config.appFqdn(app.location);
    var stdEnv = [
        'CLOUDRON=1',
        'WEBADMIN_ORIGIN=' + config.adminOrigin(),
        'API_ORIGIN=' + config.adminOrigin(),
-        'APP_ORIGIN=https://' + config.appFqdn(app.location),
-        'APP_DOMAIN=' + config.appFqdn(app.location)
+        'APP_ORIGIN=https://' + domain,
+        'APP_DOMAIN=' + domain
    ];

    // docker portBindings requires ports to be exposed
@@ -153,22 +162,28 @@ function createSubcontainer(app, name, cmd, options, callback) {
        dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
    }

-    var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
-    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
-    // this means cloudron exec does not work
-    var isolatedNetworkNs = true;
+    // first check db record, then manifest
+    var memoryLimit = app.memoryLimit || manifest.memoryLimit;
+
+    // ensure we never go below minimum
+    memoryLimit = memoryLimit < constants.DEFAULT_MEMORY_LIMIT ? constants.DEFAULT_MEMORY_LIMIT : memoryLimit; // 256mb by default
+
+    // developerMode does not restrict memory usage
+    memoryLimit = developmentMode ? 0 : memoryLimit;

    addons.getEnvironment(app, function (error, addonEnv) {
        if (error) return callback(new Error('Error getting addon environment : ' + error));

+        // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
+        // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
+        // name to look up the internal docker ip. this makes curl from within container fail
+        // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
+        // Hostname cannot be set with container NetworkMode
        var containerOptions = {
            name: name, // used for filtering logs
-            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
-            // for subcontainers, this should not be set because we already share the network namespace with app container
-            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
            Tty: isAppContainer,
            Image: app.manifest.dockerImage,
-            Cmd: cmd,
+            Cmd: (isAppContainer && developmentMode) ? [ '/bin/bash', '-c', 'echo "Development mode. Use cloudron exec to debug. Sleeping" && sleep infinity' ] : cmd,
            Env: stdEnv.concat(addonEnv).concat(portEnv),
            ExposedPorts: isAppContainer ? exposedPorts : { },
            Volumes: { // see also ReadonlyRootfs
@@ -186,23 +201,19 @@ function createSubcontainer(app, name, cmd, options, callback) {
                MemorySwap: memoryLimit, // Memory + Swap
                PortBindings: isAppContainer ? dockerPortBindings : { },
                PublishAllPorts: false,
-                ReadonlyRootfs: semver.gte(targetBoxVersion(app.manifest), '0.0.66'), // see also Volumes in startContainer
+                ReadonlyRootfs: !developmentMode, // see also Volumes in startContainer
                RestartPolicy: {
                    "Name": isAppContainer ? "always" : "no",
                    "MaximumRetryCount": 0
                },
                CpuShares: 512, // relative to 1024 for system processes
                VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
-                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
-                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
+                NetworkMode: isAppContainer ? 'cloudron' : ('container:' + app.containerId), // share network namespace with parent
                SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
            }
        };
        containerOptions = _.extend(containerOptions, options);

-        // older versions wanted a writable /var/log
-        if (semver.lte(targetBoxVersion(app.manifest), '0.0.71')) containerOptions.Volumes['/var/log'] = {};
-
        debugApp(app, 'Creating container for %s with options %j', app.manifest.dockerImage, containerOptions);

        docker.createContainer(containerOptions, callback);
@@ -329,24 +340,84 @@ function deleteImage(manifest, callback) {

    var docker = exports.connection;

-    docker.getImage(dockerImage).inspect(function (error, result) {
+    var removeOptions = {
+        force: false, // might be shared with another instance of this app
+        noprune: false // delete untagged parents
+    };
+
+    // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
+    // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
+    // not created anymore after https://github.com/docker/docker/pull/10571
+    docker.getImage(dockerImage).remove(removeOptions, function (error) {
        if (error && error.statusCode === 404) return callback(null);
+        if (error && error.statusCode === 409) return callback(null); // another container using the image

-        if (error) return callback(error);
+        if (error) debug('Error removing image %s : %j', dockerImage, error);

-        var removeOptions = {
-            force: true,
-            noprune: false
-        };
-
-         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
-        docker.getImage(result.Id).remove(removeOptions, function (error) {
-            if (error && error.statusCode === 404) return callback(null);
-            if (error && error.statusCode === 409) return callback(null); // another container using the image
-
-            if (error) debug('Error removing image %s : %j', dockerImage, error);
-
-            callback(error);
-        });
+        callback(error);
    });
 }
+
+function getContainerIdByIp(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('get container by ip %s', ip);
+
+    var docker = exports.connection;
+
+    docker.listNetworks({}, function (error, result) {
+        if (error) return callback(error);
+
+        var bridge;
+        result.forEach(function (n) {
+            if (n.Name === 'cloudron') bridge = n;
+        });
+
+        if (!bridge) return callback(new Error('Unable to find the cloudron network'));
+
+        var containerId;
+        for (var id in bridge.Containers) {
+            if (bridge.Containers[id].IPv4Address.indexOf(ip) === 0) {
+                containerId = id;
+                break;
+            }
+        }
+        if (!containerId) return callback(new Error('No container with that ip'));
+
+        debug('found container %s with ip %s', containerId, ip);
+
+        callback(null, containerId);
+    });
+}
+
+function execContainer(containerId, cmd, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert(util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', '-i', containerId  ].concat(cmd));
+
+    var chunks = [ ];
+
+    if (options.stdout) {
+        cp.stdout.pipe(options.stdout);
+    } else if (options.bufferStdout) {
+        cp.stdout.on('data', function (chunk) { chunks.push(chunk); });
+    } else {
+        cp.stdout.pipe(process.stdout);
+    }
+
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debug('execContainer code: %s signal: %s', code, signal);
+        if (!callback.called) callback(code ? 'Failed with status ' + code : null, Buffer.concat(chunks));
+    });
+
+    cp.stderr.pipe(options.stderr || process.stderr);
+
+    if (options.stdin) options.stdin.pipe(cp.stdin).on('error', callback);
+}
@@ -0,0 +1,116 @@
+'use strict';
+
+exports = module.exports = {
+    EventLogError: EventLogError,
+
+    add: add,
+    get: get,
+    getAllPaged: getAllPaged,
+    cleanup: cleanup,
+
+    // keep in sync with webadmin index.js filter
+    ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CLONE: 'app.clone',
+    ACTION_APP_CONFIGURE: 'app.configure',
+    ACTION_APP_INSTALL: 'app.install',
+    ACTION_APP_RESTORE: 'app.restore',
+    ACTION_APP_UNINSTALL: 'app.uninstall',
+    ACTION_APP_UPDATE: 'app.update',
+    ACTION_BACKUP_FINISH: 'backup.finish',
+    ACTION_BACKUP_START: 'backup.start',
+    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CLI_MODE: 'settings.climode',
+    ACTION_START: 'cloudron.start',
+    ACTION_UPDATE: 'cloudron.update',
+    ACTION_USER_ADD: 'user.add',
+    ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_REMOVE: 'user.remove',
+    ACTION_USER_UPDATE: 'user.update'
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:eventlog'),
+    eventlogdb = require('./eventlogdb.js'),
+    util = require('util'),
+    uuid = require('node-uuid');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function EventLogError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(EventLogError, Error);
+EventLogError.INTERNAL_ERROR = 'Internal error';
+EventLogError.NOT_FOUND = 'Not Found';
+
+function add(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    var id = uuid.v4();
+
+    eventlogdb.add(id, action, source, data, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, { id: id });
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new EventLogError(EventLogError.NOT_FOUND, 'No such event'));
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.getAllPaged(action, search, page, perPage, function (error, boxes) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, boxes);
+    });
+}
+
+function cleanup(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+     var d = new Date();
+     d.setDate(d.getDate() - 7); // 7 days ago
+
+    eventlogdb.delByCreationTime(d, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
@@ -0,0 +1,116 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getAllPaged: getAllPaged,
+    add: add,
+    count: count,
+    delByCreationTime: delByCreationTime,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    mysql = require('mysql'),
+    safe = require('safetydance'),
+    util = require('util');
+
+var EVENTLOGS_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+
+// until mysql module supports automatic type coercion
+function postProcess(eventLog) {
+    eventLog.source = safe.JSON.parse(eventLog.source);
+    eventLog.data = safe.JSON.parse(eventLog.data);
+    return eventLog;
+}
+
+function get(eventId, callback) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE id = ?', [ eventId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, postProcess(result[0]));
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [];
+    var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog';
+
+    if (action || search) query += ' WHERE';
+    if (search) query += ' data LIKE ' + mysql.escape('%' + search + '%');
+    if (action && search) query += ' AND ';
+
+    if (action) {
+        query += ' action=?';
+        data.push(action);
+    }
+
+    query += ' ORDER BY creationTime DESC LIMIT ?,?';
+
+    data.push((page-1)*perPage);
+    data.push(perPage);
+
+    database.query(query, data, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, action, source, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM eventlog', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM eventlog', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
+
+function delByCreationTime(creationTime, callback) {
+    assert(util.isDate(creationTime));
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM eventlog WHERE creationTime < ?', [ creationTime ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
@@ -0,0 +1,216 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
+    add: add,
+    del: del,
+    count: count,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror');
+
+var GROUPS_FIELDS = [ 'id', 'name' ].join(',');
+
+function get(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups WHERE id = ?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result[0]);
+    });
+}
+
+function getWithMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' WHERE groups.id = ? ' +
+                    ' GROUP BY groups.id', [ groupId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        var result = results[0];
+        result.userIds = result.userIds ? result.userIds.split(',') : [ ];
+
+        callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' GROUP BY groups.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
+function add(id, name, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [ id, name ];
+    database.query('INSERT INTO groups (id, name) VALUES (?, ?)',
+           data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
+    queries.push({ query: 'DELETE FROM groups WHERE id = ?', args: [ id ] });
+
+    database.transaction(queries, function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(error);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM groupMembers', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        database.query('DELETE FROM groups WHERE id != ?', [ 'admin' ], function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            callback(error);
+        });
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.userId; }));
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.groupId; }));
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [ ];
+    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
+    groupIds.forEach(function (gid) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result.length !== 0);
+    });
+}
@@ -0,0 +1,221 @@
+/* jshint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    GroupError: GroupError,
+
+    create: create,
+    remove: remove,
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    ADMIN_GROUP_ID: 'admin' // see db migration code and groupdb._clear
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    groupdb = require('./groupdb.js'),
+    util = require('util');
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function GroupError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(GroupError, Error);
+GroupError.INTERNAL_ERROR = 'Internal Error';
+GroupError.ALREADY_EXISTS = 'Already Exists';
+GroupError.NOT_FOUND = 'Not Found';
+GroupError.BAD_FIELD = 'Field error';
+GroupError.NOT_EMPTY = 'Not Empty';
+GroupError.NOT_ALLOWED = 'Not Allowed';
+
+function validateGroupname(name) {
+    assert.strictEqual(typeof name, 'string');
+    var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
+
+    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');
+
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');
+
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
+
+    return null;
+}
+
+function create(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateGroupname(name);
+    if (error) return callback(error);
+
+    groupdb.add(name /* id */, name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new GroupError(GroupError.ALREADY_EXISTS));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null, { id: name, name: name });
+    });
+}
+
+function remove(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // never allow admin group to be deleted
+    if (id === exports.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
+
+    groupdb.del(id, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getWithMembers(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getWithMembers(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAll(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getMembers(groupId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getGroups(userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.setGroups(userId, groupIds, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.addMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.removeMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.isMember(groupId, userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
@@ -0,0 +1,22 @@
+'use strict';
+
+// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+// These constants are used in the installer script as well
+// Do not require anything here!
+
+exports = module.exports = {
+    // a version bump means that all containers (apps and addons) are recreated
+    'version': 40,
+
+    'baseImage': 'cloudron/base:0.8.1',
+
+    'images': {
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.12.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.11.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.10.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.9.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.18.0' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.9.0' }
+    }
+};
+
@@ -6,11 +6,15 @@ exports = module.exports = {
 };

 var assert = require('assert'),
+    apps = require('./apps.js'),
    config = require('./config.js'),
    debug = require('debug')('box:ldap'),
+    eventlog = require('./eventlog.js'),
    user = require('./user.js'),
    UserError = user.UserError,
-    ldap = require('ldapjs');
+    ldap = require('ldapjs'),
+    mailboxes = require('./mailboxes.js'),
+    MailboxError = mailboxes.MailboxError;

 var gServer = null;

@@ -28,105 +32,238 @@ var gLogger = {
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
 var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';

+function getAppByRequest(req, callback) {
+    var sourceIp = req.connection.ldap.id.split(':')[0];
+    if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
+
+    apps.getByIpAddress(sourceIp, function (error, app) {
+        if (error) return callback(new ldap.OperationsError(error.message));
+
+        if (!app) return callback(new ldap.OperationsError('Could not detect app source'));
+
+        callback(null, app);
+    });
+}
+
+function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // send user objects
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+            var groups = [ GROUP_USERS_DN ];
+            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+            var displayName = entry.displayName || entry.username;
+            var nameParts = displayName.split(' ');
+            var firstName = nameParts[0];
+            var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['user'],
+                    objectcategory: 'person',
+                    cn: entry.id,
+                    uid: entry.id,
+                    mail: entry.email,
+                    // TODO: check mailboxes before we send this
+                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    displayname: displayName,
+                    givenName: firstName,
+                    username: entry.username,
+                    samaccountname: entry.username,      // to support ActiveDirectory clients
+                    memberof: groups
+                }
+            };
+
+            // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+            // which is required to have atleast one character if present
+            if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result){
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        var groups = [{
+            name: 'users',
+            admin: false
+        }, {
+            name: 'admins',
+            admin: true
+        }];
+
+        groups.forEach(function (group) {
+            var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: group.name,
+                    memberuid: members.map(function(entry) { return entry.id; })
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function mailboxSearch(req, res, next) {
+    debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
+
+            // TODO: send aliases
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['mailbox'],
+                    objectcategory: 'mailbox',
+                    cn: entry.name,
+                    uid: entry.name,
+                    mail: entry.name + '@' + config.fqdn()
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function authenticateUser(req, res, next) {
+    debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+    // extract the common name which might have different attribute names
+    var attributeName = Object.keys(req.dn.rdns[0])[0];
+    var commonName = req.dn.rdns[0][attributeName];
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    var api;
+    if (attributeName === 'mail') {
+        api = user.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        var parts = commonName.split('@');
+        if (parts[1] === config.fqdn()) { // internal email, verify with username
+            commonName = parts[0];
+            api = user.verifyWithUsername;
+        } else { // external email
+            api = user.verifyWithEmail;
+        }
+    } else if (commonName.indexOf('uid-') === 0) {
+        api = user.verify;
+    } else {
+        api = user.verifyWithUsername;
+    }
+
+    api(commonName, req.credentials || '', function (error, user) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorizeUserForApp(req, res, next) {
+    assert(req.user);
+
+    getAppByRequest(req, function (error, app) {
+        if (error) return next(error);
+
+        apps.hasAccessTo(app, req.user, function (error, result) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we return no such object, to avoid leakage of a users existence
+            if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: req.user.id });
+
+            res.end();
+        });
+    });
+}
+
+function authorizeUserForMailbox(req, res, next) {
+    assert(req.user);
+
+    // We simply authorize the user to access a mailbox by his own name
+    mailboxes.get(req.user.username, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+
+        res.end();
+    });
+}
+
 function start(callback) {
    assert.strictEqual(typeof callback, 'function');

    gServer = ldap.createServer({ log: gLogger });

-    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+    gServer.search('ou=users,dc=cloudron', userSearch);
+    gServer.search('ou=groups,dc=cloudron', groupSearch);
+    gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);

-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);

-            // send user objects
-            result.forEach(function (entry) {
-                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
-
-                var groups = [ GROUP_USERS_DN ];
-                if (entry.admin) groups.push(GROUP_ADMINS_DN);
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['user'],
-                        objectcategory: 'person',
-                        cn: entry.id,
-                        uid: entry.id,
-                        mail: entry.email,
-                        displayname: entry.username,
-                        username: entry.username,
-                        samaccountname: entry.username,      // to support ActiveDirectory clients
-                        memberof: groups
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
-
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
-
-            var groups = [{
-                name: 'users',
-                admin: false
-            }, {
-                name: 'admins',
-                admin: true
-            }];
-
-            groups.forEach(function (group) {
-                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['group'],
-                        cn: group.name,
-                        memberuid: members.map(function(entry) { return entry.id; })
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
-        // TODO: validate password
-        debug('ldap application bind: %s', req.dn.toString());
+    // this is the bind for addons (after bind, they might search and authenticate)
+    gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
+        debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
        res.end();
    });

-    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('ldap user bind: %s', req.dn.toString());
-
-        if (!req.dn.rdns[0].cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-        user.verify(req.dn.rdns[0].cn, req.credentials || '', function (error, result) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-            if (error) return next(new ldap.OperationsError(error));
-
-            res.end();
-        });
+    // this is the bind for apps (after bind, they might search and authenticate user)
+    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
+        // TODO: validate password
+        debug('application bind: %s', req.dn.toString());
+        res.end();
    });

-    gServer.listen(config.get('ldapPort'), callback);
+    gServer.listen(config.get('ldapPort'), '0.0.0.0', callback);
 }

 function stop(callback) {
@@ -0,0 +1,37 @@
+'use strict';
+
+exports = module.exports = {
+    sendFailureLogs: sendFailureLogs
+};
+
+var assert = require('assert'),
+    mailer = require('./mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
+
+function collectLogs(unitName, callback) {
+    assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendFailureLogs(processName, options) {
+    assert.strictEqual(typeof processName, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    collectLogs(options.unit || processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending failure logs for', processName);
+
+        mailer.unexpectedExit(processName, result);
+    });
+}
@@ -5,8 +5,7 @@ Dear Admin,
 The application titled '<%= title %>' that you installed at <%= appFqdn %>
 is not responding.

-This is most likely a problem in the application. Please report this issue to
-support@cloudron.io (by forwarding this email).
+This is most likely a problem in the application.

 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.

@@ -2,12 +2,15 @@

 Dear Admin,

-A new version of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+A new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!

-Please update at your convenience at <%= webadminUrl %>.
+The app will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
+
+Changes:
+<%= updateInfo.manifest.changelog %>

 Thank you,
-Update Manager
+your Cloudron

 <% } else { %>

@@ -2,9 +2,9 @@

 Dear Admin,

-A new version of Cloudron <%= fqdn %> is available!
+Version <%= newBoxVersion %> of Cloudron <%= fqdn %> is now available!

-Please update at your convenience at <%= webadminUrl %>.
+Your Cloudron will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.

 Changelog:
 <% for (var i = 0; i < changelog.length; i++) { %>
@@ -12,7 +12,7 @@ Changelog:
 <% } %>

 Thank you,
-Update Manager
+your Cloudron

 <% } else { %>

@@ -0,0 +1,16 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+<% if (message) { %>
+<%= domain %> was not renewed.
+
+<%- message %>
+<% } else { %>
+<%= domain %> was renewed.
+<% } %>
+Thank you,
+Your Cloudron
+<% } else { %>
+
+<% } %>
+
@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+
+<%= fqdn %> is running out of disk space.
+
+Please see some excerpts of the logs below.
+
+Thank you,
+Your Cloudron
+
+-------------------------------------
+
+<%- message %>
+
+<% } else { %>
+
+<% } %>
+
@@ -1,6 +1,6 @@
 <%if (format === 'text') { %>

-Dear <%= username %>,
+Dear <%= user.username || user.email %>,

 Someone, hopefully you, has requested your <%= fqdn %>'s account password
 be reset. If you did not request this reset, please ignore this message.
@@ -2,7 +2,7 @@

 Dear Cloudron Team,

-Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> exited unexpectedly!

 Please see some excerpt of the logs below.

@@ -0,0 +1,23 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+User with email <%= user.email %> was added in the Cloudron at <%= fqdn %>.
+
+You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
+
+<% if (inviteLink) { %>
+As requested, this user has not been sent an invitation email.
+
+To set a password and perform any configuration on behalf of the user, please use this link:
+<%= inviteLink %>
+
+<% } %>
+
+Thank you,
+User Manager
+
+<% } else { %>
+
+<% } %>
+
@@ -2,7 +2,7 @@

 Dear Admin,

-User with name '<%= username %>' (<%= email %>) <%= event %> in the Cloudron at <%= fqdn %>.
+User <%= user.username %> <%= user.email %> <%= event %> in the Cloudron at <%= fqdn %>.

 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.

@@ -1,22 +1,21 @@
 <%if (format === 'text') { %>

-Dear <%= user.username %>,
+Dear <%= user.email %>,

-Welcome to my Cloudron <%= fqdn %>!
+Welcome to our Cloudron <%= fqdn %>!

 The Cloudron is our own Smart Server. You can read more about it
 at https://www.cloudron.io.

-           You username is '<%= user.username %>'
-
 To get started, create your account by visiting the following page:
 <%= setupLink %>

-When you visit the above page, you will be prompted to enter a new password.
-After you have submitted the form, you can login using the new password.
-
+<% if (invitor && invitor.email) { %>
 Thank you,
 <%= invitor.email %>
+<% } else { %>
+Thank you
+<% } %>

 <% } else { %>

@@ -0,0 +1,127 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    getAliases: getAliases,
+    setAliases: setAliases,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'aliasTarget', 'creationTime' ].join(',');
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name) VALUES (?)', [ name ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE name=? OR aliasTarget = ?', [ name, name ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function postProcess(result) {
+    result.aliases = result.aliases ? result.aliases.split(',') : [ ];
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.name=? AND m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(results[0]);
+
+        callback(null, results[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ?', args: [ name ] });
+    aliases.forEach(function (alias) {
+        queries.push({ query: 'INSERT INTO mailboxes (name, aliasTarget) VALUES (?, ?)', args: [ alias, name ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name FROM mailboxes WHERE aliasTarget=? ORDER BY name', [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results = results.map(function (r) { return r.name; });
+        callback(null, results);
+    });
+}
@@ -0,0 +1,199 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    setAliases: setAliases,
+    getAliases: getAliases,
+
+    setupAliases: setupAliases,
+
+    MailboxError: MailboxError
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:mailboxes'),
+    docker = require('./docker.js'),
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util');
+
+function MailboxError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(MailboxError, Error);
+MailboxError.ALREADY_EXISTS = 'already exists';
+MailboxError.BAD_FIELD = 'Field error';
+MailboxError.NOT_FOUND = 'not found';
+MailboxError.INTERNAL_ERROR = 'internal error';
+MailboxError.EXTERNAL_ERROR = 'external error';
+
+function validateName(name) {
+    var RESERVED_NAMES = [ 'no-reply', 'postmaster', 'mailer-daemon' ];
+
+    if (!name.length) return new MailboxError(MailboxError.BAD_FIELD, "name cannot be empty");
+
+    if (name.length < 2) return new MailboxError(MailboxError.BAD_FIELD, 'name too small');
+    if (name.length > 127) return new MailboxError(MailboxError.BAD_FIELD, 'name too long');
+    if (RESERVED_NAMES.indexOf(name) !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'name is reserved');
+
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailboxError(MailboxError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    if (name.indexOf('.app') !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'alias pattern is reserved for apps');
+
+    return null;
+}
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.add(name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        debug('Added mailbox %s', name);
+
+        var mailbox = {
+            name: name
+        };
+
+        callback(null, mailbox);
+    });
+}
+
+function pushAlias(name, aliases, callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    var cmd = [ '/addons/mail/service.sh', 'set-alias', name ].concat(aliases);
+
+    debug('pushing alias for %s : %j', name, aliases);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(new MailboxError(MailboxError.EXTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    pushAlias(name, [ ], function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.del(name, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+            debug('deleted mailbox %s', name);
+
+            callback();
+        });
+    });
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.get(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, mailbox);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAll(function (error, results) {
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var i = 0; i < aliases.length; i++) {
+        aliases[i] = aliases[i].toLowerCase();
+
+        var error = validateName(aliases[i]);
+        if (error) return callback(error);
+    }
+
+    pushAlias(name, aliases, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.setAliases(name, aliases, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS, error.message))
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+
+            callback(null);
+        });
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAliases(name, function (error, aliases) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, aliases);
+    });
+}
+
+// push aliases to the mail container on startup
+function setupAliases(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, mailboxes) {
+        if (error) return callback(error);
+
+        async.each(mailboxes, function (mailbox, iteratorDone) {
+            getAliases(mailbox.name, function (error, aliases) {
+                if (error) return iteratorDone(error);
+
+                if (aliases.length === 0) return iteratorDone();
+
+                pushAlias(mailbox.name, aliases, iteratorDone);
+            });
+        }, callback);
+    });
+}
+
--- a/Show More
+++ b/Show More