more changes

Fix autoupdate detection logic
We should be comparing existing manifest ports with new manifest ports. The user chosen bindings itself doesn't matter.
2016-06-08 08:46:18 -07:00 · 2016-06-08 08:45:40 -07:00 · 2016-06-08 00:07:41 -07:00 · 2016-06-07 20:57:39 -07:00 · 2016-06-07 20:24:41 -07:00 · 2016-06-07 20:09:53 -07:00
172 changed files with 11981 additions and 5520 deletions
@@ -424,3 +424,106 @@
 - Admin flag is handled outside of groups
 - User interface fixes for groups
 - Allow to set access restrictions on app installation
+
+[0.10.0]
+- Upgrade to docker 1.10.2
+- Fix MySQL addon to handle heavier loads
+- Allow listing and download of backups (using the CLI tool)
+- Ubuntu security updates till 8th March 2016 (http://www.ubuntu.com/usn)
+
+[0.10.1]
+- Fix Let's Encrypt certificate renewal
+
+[0.10.2]
+- Apps can now bind with username or email with LDAP
+- Disallow updating an app with mismatching manifest id
+- Use admin domain instead of naked domain in the SPF record
+- Download Lets Encrypt intermediate cert
+
+[0.10.3]
+- Store the backup config for each backup. This will allow using multiple buckets/providers for backups simultaneously.
+- Fix SPF record check
+
+[0.10.4]
+- Fix restore for droplets in EU region
+
+[0.11.0]
+- Store backups in the same region as the Cloudron
+- Fix PCRE security issue (http://www.ubuntu.com/usn/usn-2943-1/)
+
+[0.11.1]
+- Improve the backup logic
+
+[0.11.2]
+- Allow users to choose a username on first sign up
+- Fix app graphs
+
+[0.12.0]
+- Fix upload of large backups
+- Postgres addon whitelists pg_trgm and hstore extensions
+- Suppress boring update emails from patch releases
+- Setup bounce alerts for emails
+- Query admin's name in activation wizard
+- Admin emails are now delivered as no-reply
+- Fix crash when user attempts to set a duplicate email
+- Improved mongodb crash recovery
+
+[0.12.1]
+- Fix crash when backing up apps
+
+[0.12.2]
+- Improved error handling for addons
+
+[0.12.3]
+- LDAP: Do not set sn attribute when user has no surname
+
+[0.12.4]
+- Install app only after platform is ready
+
+[0.12.5]
+- Get alerts for app task failures
+- Fix update issue when one or more apps are in failed state
+
+[0.12.6]
+- Allow setting an alternate external domain for apps
+
+[0.12.7]
+- Fix changing password
+
+[0.13.0]
+- Upgrade to ubuntu 16.04
+- Add event log
+
+[0.13.1]
+- Make activity log viewable to admins
+- Fix geoip lookup
+
+[0.13.2]
+- Fix crash in app auto updater
+- Fix crash with empty timezone
+
+[0.13.3]
+- Enable auth in email addon
+- Add search for activity log
+- Add tutorial for first time users
+
+[0.13.4]
+- Fix mail addon restart issue
+
+[0.14.0]
+- You have mail :-)
+
+[0.14.1]
+- 2-character usernames are now allowed
+- Make cloudron CLI push/pull more robust
+
+[0.14.2]
+- Update mail addon
+
+[0.15.0]
+- [REST API](https://cloudron.io/references/api.html) is now in public beta
+- Enable Developer mode by default for new Cloudrons
+- Reverse proxy fixes for apps exposing a WebDav server
+- Allow admins to optionall set the username and displayName on user creation
+- Fix app autoupdate logic to detect if one or more in-use port bindings was removed
+
@@ -137,8 +137,8 @@ while true; do
    sleep 30
 done

-echo "Copying INFRA_VERSION"
-$scp22 "${SCRIPT_DIR}/../setup/INFRA_VERSION" root@${server_ip}:.
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" root@${server_ip}:.

 echo "Copying box source"
 cd "${SOURCE_DIR}"
@@ -10,7 +10,7 @@ if [[ -z "${JSON}" ]]; then
    exit 1
 fi

-readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"

 function debug() {
    echo "$@" >&2
@@ -30,7 +30,7 @@ function create_droplet() {
    local box_name="$2"

    local image_region="sfo1"
-    local ubuntu_image_slug="ubuntu-15-10-x64"
+    local ubuntu_image_slug="ubuntu-16-04-x64"
    local box_size="512mb"

    local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
@@ -109,13 +109,23 @@ function get_image_id() {
    local snapshot_name="$1"
    local image_id=""

-    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
-
-    if [[ -n "${image_id}" ]]; then
-        echo "${image_id}"
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=100"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
    fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
 }

 function snapshot_droplet() {
@@ -128,16 +138,26 @@ function snapshot_droplet() {
    debug -n "Waiting for snapshot to complete"

    while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
        if [[ "${event_status}" == "completed" ]]; then
            break
        fi
        debug -n "."
        sleep 10
    done
-    debug ""
+    debug "! done"

-    get_image_id "${snapshot_name}"
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
 }

 function destroy_droplet() {
@@ -17,13 +17,7 @@ function die {
    exit 1
 }

-[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
-
-if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
-    source "${SOURCE_DIR}/INFRA_VERSION"
-else
-    echo "No INFRA_VERSION found, skip pulling docker images"
-fi
+[[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"

 if [ ${SELFHOSTED} == 0 ]; then
    echo "!! Initializing Ubuntu image for CaaS"
@@ -49,7 +43,7 @@ export DEBIAN_FRONTEND=noninteractive

 echo "=== Upgrade ==="
 apt-get update
-apt-get upgrade -y
+apt-get dist-upgrade -y
 apt-get install -y curl

 # Setup firewall before everything. docker creates it's own chain and the -X below will remove it
@@ -67,9 +61,9 @@ iptables -P OUTPUT ACCEPT
 # allow ssh, http, https, ping, dns
 iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
 if [ ${SELFHOSTED} == 0 ]; then
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
 else
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,22,443,587,993,4190 -j ACCEPT
 fi
 iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
@@ -94,7 +88,8 @@ apt-get -y install btrfs-tools

 echo "==== Install docker ===="
 # install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
-curl https://get.docker.com/builds/Linux/x86_64/docker-1.9.1 > /usr/bin/docker
+curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
+apt-get -y install aufs-tools
 chmod +x /usr/bin/docker
 groupadd docker
 cat > /etc/systemd/system/docker.socket <<EOF
@@ -129,10 +124,10 @@ WantedBy=multi-user.target
 EOF

 echo "=== Setup btrfs data ==="
-fallocate -l "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
+truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
 mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
-echo "${USER_DATA_FILE} ${USER_DATA_DIR} btrfs loop,nosuid 0 0" >> /etc/fstab
-mkdir -p "${USER_DATA_DIR}" && mount "${USER_DATA_FILE}"
+mkdir -p "${USER_DATA_DIR}"
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}

 systemctl daemon-reload
 systemctl enable docker
@@ -155,34 +150,26 @@ update-grub
 # now add the user to the docker group
 usermod "${USER}" -a -G docker

-if [ -z $(echo "${INFRA_VERSION}") ]; then
-    echo "Skip pulling base docker images"
-else
-    echo "=== Pulling base docker images ==="
-    docker pull "${BASE_IMAGE}"
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"

-    echo "=== Pulling mysql addon image ==="
-    docker pull "${MYSQL_IMAGE}"
+echo "==== Downloading docker images ===="
+images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage); console.log(Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join('\n'));")

-    echo "=== Pulling postgresql addon image ==="
-    docker pull "${POSTGRESQL_IMAGE}"
-
-    echo "=== Pulling redis addon image ==="
-    docker pull "${REDIS_IMAGE}"
-
-    echo "=== Pulling mongodb addon image ==="
-    docker pull "${MONGODB_IMAGE}"
-
-    echo "=== Pulling graphite docker images ==="
-    docker pull "${GRAPHITE_IMAGE}"
-
-    echo "=== Pulling mail relay ==="
-    docker pull "${MAIL_IMAGE}"
-fi
+echo "Pulling images: ${images}"
+for image in images; do
+    docker pull "${image}"
+done

 echo "==== Install nginx ===="
 apt-get -y install nginx-full
-[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
+[[ "$(nginx -v 2>&1)" == *"nginx/1.10."* ]] || die "Expecting nginx version to be 1.10.x"

 echo "==== Install build-essential ===="
 apt-get -y install build-essential rcconf
@@ -190,11 +177,11 @@ apt-get -y install build-essential rcconf
 echo "==== Install mysql ===="
 debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
 debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-apt-get -y install mysql-server
-[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
+apt-get -y install mysql-server-5.7
+[[ "$(mysqld --version 2>&1)" == *"5.7."* ]] || die "Expecting mysql version to be 5.7.x"

-echo "==== Install pwgen ===="
-apt-get -y install pwgen
+echo "==== Install pwgen and swaks awscli ===="
+apt-get -y install pwgen swaks awscli

 echo "==== Install collectd ==="
 if ! apt-get install -y collectd collectd-utils; then
@@ -209,15 +196,6 @@ echo "==== Install logrotate ==="
 apt-get install -y cron logrotate
 systemctl enable cron

-echo "==== Install nodejs ===="
-# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
-mkdir -p /usr/local/node-4.1.1
-curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
-ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
-ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
 echo "=== Rebuilding npm packages ==="
 cd "${INSTALLER_SOURCE_DIR}" && npm install --production
 chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
@@ -268,7 +246,7 @@ echo "==== Install box-setup systemd script ===="
 cat > /etc/systemd/system/box-setup.service <<EOF
 [Unit]
 Description=Box Setup
-Before=docker.service collectd.service
+Before=docker.service collectd.service mysql.service
 After=do-resize.service

 [Service]
@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();
@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendFailureLogs(processName, { unit: processName });
+}
+
+main();
@@ -22,6 +22,7 @@ gulp.task('3rdparty', function () {
        'webadmin/src/3rdparty/**/*.otf',
        'webadmin/src/3rdparty/**/*.eot',
        'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.gif',
        'webadmin/src/3rdparty/**/*.ttf',
        'webadmin/src/3rdparty/**/*.woff',
        'webadmin/src/3rdparty/**/*.woff2'
@@ -4,7 +4,6 @@ set -eu -o pipefail

 readonly USER_HOME="/home/yellowtent"
 readonly APPS_SWAP_FILE="/apps.swap"
-readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"

@@ -23,7 +22,6 @@ readonly swap_size="${physical_memory}" # if you change this, fix enoughResource
 readonly app_count=$((${physical_memory} / 200)) # estimated app count
 readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
 readonly disk_size=$((disk_size_gb * 1024))
-readonly backup_swap_size=1024
 readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
 readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1

@@ -32,8 +30,7 @@ echo "Physical memory: ${physical_memory}"
 echo "Estimated app count: ${app_count}"
 echo "Disk size: ${disk_size}"

-# Allocate two sets of swap files - one for general app usage and another for backup
-# The backup swap is setup for swap on the fly by the backup scripts
+# Allocate swap for general app usage
 if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
    echo "Creating Apps swap file of size ${swap_size}M"
    fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
@@ -45,22 +42,13 @@ else
    echo "Apps Swap file already exists"
 fi

-if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then
-    echo "Creating Backup swap file of size ${backup_swap_size}M"
-    fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}"
-    chmod 600 "${BACKUP_SWAP_FILE}"
-    mkswap "${BACKUP_SWAP_FILE}"
-else
-    echo "Backups Swap file already exists"
-fi
-
 echo "Resizing data volume"
-home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
+home_data_size=$((disk_size - system_size - swap_size - ext4_reserved))
 echo "Resizing up btrfs user data to size ${home_data_size}M"
-umount "${USER_DATA_DIR}"
+umount "${USER_DATA_DIR}" || true
 # Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data
 # fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
 truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
-mount "${USER_DATA_FILE}"
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
 btrfs filesystem resize max "${USER_DATA_DIR}"

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE backups(" +
+            "filename VARCHAR(128) NOT NULL," +
+            "creationTime TIMESTAMP," +
+            "version VARCHAR(128) NOT NULL," +
+            "type VARCHAR(16) NOT NULL," +
+            "dependsOn VARCHAR(4096)," +
+            "state VARCHAR(16) NOT NULL," +
+            "PRIMARY KEY (filename))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE backups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
@@ -0,0 +1,17 @@
+var dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE filename id VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE id filename VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,17 @@
+'use strict';
+
+var dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN altDomain VARCHAR(256)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN altDomain', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,24 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE eventlog(" +
+            "id VARCHAR(128) NOT NULL," +
+            "source JSON," +
+            "creationTime TIMESTAMP," +
+            "action VARCHAR(128) NOT NULL," +
+            "data JSON," +
+            "PRIMARY KEY (id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE eventlog', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN showTutorial BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN showTutorial', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,24 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};
@@ -11,7 +11,7 @@

 CREATE TABLE IF NOT EXISTS users(
    id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(254) NOT NULL UNIQUE,
+    username VARCHAR(254) UNIQUE,
    email VARCHAR(254) NOT NULL UNIQUE,
    password VARCHAR(1024) NOT NULL,
    salt VARCHAR(512) NOT NULL,
@@ -19,6 +19,7 @@ CREATE TABLE IF NOT EXISTS users(
    modifiedAt VARCHAR(512) NOT NULL,
    admin INTEGER NOT NULL,
    displayName VARCHAR(512) DEFAULT '',
+    showTutorial BOOLEAN DEFAULT 0,
    PRIMARY KEY(id));

 CREATE TABLE IF NOT EXISTS groups(
@@ -37,7 +38,7 @@ CREATE TABLE IF NOT EXISTS tokens(
    identifier VARCHAR(128) NOT NULL,
    clientId VARCHAR(128),
    scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires BIGINT NOT NULL, // FIXME: make this a timestamp
    PRIMARY KEY(accessToken));

 CREATE TABLE IF NOT EXISTS clients(
@@ -61,10 +62,11 @@ CREATE TABLE IF NOT EXISTS apps(
    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
    location VARCHAR(128) NOT NULL UNIQUE,
    dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
    oauthProxy BOOLEAN DEFAULT 0,
    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
    memoryLimit BIGINT DEFAULT 0,
+    altDomain VARCHAR(256),

    lastBackupId VARCHAR(128),
    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
@@ -84,7 +86,7 @@ CREATE TABLE IF NOT EXISTS authcodes(
    authCode VARCHAR(128) NOT NULL UNIQUE,
    userId VARCHAR(128) NOT NULL,
    clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
    PRIMARY KEY(authCode));

 CREATE TABLE IF NOT EXISTS settings(
@@ -98,3 +100,32 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
    value VARCHAR(512) NOT NULL,
    FOREIGN KEY(appId) REFERENCES apps(id));

+CREATE TABLE IF NOT EXISTS backups(
+    filename VARCHAR(128) NOT NULL,
+    creationTime TIMESTAMP,
+    version VARCHAR(128) NOT NULL, /* app version or box version */
+    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
+    dependsOn VARCHAR(4096), /* comma separate list of objects this backup depends on */
+    state VARCHAR(16) NOT NULL,
+
+    PRIMARY KEY (filename));
+
+CREATE TABLE IF NOT EXISTS eventlog(
+    id VARCHAR(128) NOT NULL,
+    action VARCHAR(128) NOT NULL,
+    source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+
+    PRIMARY KEY (id));
+
+/* Future fields:
+   * accessRestriction - to determine who can access it. So this has foreign keys
+   * quota - per mailbox quota
+*/
+CREATE TABLE IF NOT EXISTS mailboxes(
+    name VARCHAR(128) NOT NULL,
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
+    creationTime TIMESTAMP,
+
+    PRIMARY KEY (id));
@@ -14,13 +14,11 @@
  ],
  "dependencies": {
    "async": "^1.2.1",
-    "attempt": "^1.0.1",
    "aws-sdk": "^2.1.46",
    "body-parser": "^1.13.1",
-    "bytes": "^2.1.0",
-    "cloudron-manifestformat": "^2.3.0",
+    "cloudron-manifestformat": "^2.4.0",
    "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
+    "connect-lastmile": "^0.1.0",
    "connect-timeout": "^1.5.0",
    "cookie-parser": "^1.3.5",
    "cookie-session": "^1.1.0",
@@ -28,17 +26,17 @@
    "csurf": "^1.6.6",
    "db-migrate": "^0.9.2",
    "debug": "^2.2.0",
-    "dockerode": "^2.2.2",
+    "dockerode": "^2.2.10",
    "ejs": "^2.2.4",
-    "ejs-cli": "^1.0.1",
+    "ejs-cli": "^1.2.0",
    "express": "^4.12.4",
    "express-session": "^1.11.3",
    "hat": "0.0.3",
+    "ini": "^1.3.4",
    "json": "^9.0.3",
    "ldapjs": "^0.7.1",
-    "memorystream": "^0.3.0",
    "mime": "^1.3.4",
-    "morgan": "^1.6.0",
+    "morgan": "^1.7.0",
    "multiparty": "^4.1.2",
    "mysql": "^2.7.0",
    "native-dns": "^0.7.0",
@@ -48,6 +46,7 @@
    "nodemailer-smtp-transport": "^1.0.3",
    "oauth2orize": "^1.0.1",
    "once": "^1.3.2",
+    "parse-links": "^0.1.0",
    "passport": "^0.2.2",
    "passport-http": "^0.2.2",
    "passport-http-bearer": "^1.0.1",
@@ -55,18 +54,18 @@
    "passport-oauth2-client-password": "^0.1.2",
    "password-generator": "^2.0.2",
    "proxy-middleware": "^0.13.0",
-    "safetydance": "^0.1.0",
+    "safetydance": "^0.1.1",
    "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
    "split": "^1.0.0",
-    "superagent": "^1.5.0",
+    "superagent": "^1.8.3",
    "supererror": "^0.7.1",
    "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "tldjs": "^1.6.2",
    "underscore": "^1.7.0",
-    "ursa": "^0.9.1",
+    "ursa": "^0.9.3",
    "valid-url": "^1.0.9",
-    "validator": "^4.4.0",
-    "x509": "^0.2.2"
+    "validator": "^4.9.0",
+    "x509": "^0.2.4"
  },
  "devDependencies": {
    "apidoc": "*",
@@ -89,7 +88,6 @@
    "mocha": "*",
    "nock": "^3.4.0",
    "node-sass": "^3.0.0-alpha.0",
-    "redis": "^2.4.2",
    "request": "^2.65.0",
    "sinon": "^1.12.2",
    "yargs": "^3.15.0"
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=23
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.10.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.8.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.9.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite
@@ -21,6 +21,7 @@ arg_backup_config=""
 arg_dns_config=""
 arg_update_config=""
 arg_provider=""
+arg_app_bundle=""

 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -37,6 +38,9 @@ while true; do
        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
 EOF
        # read possibly empty parameters here
+        arg_app_bundle=$(echo "$2" | $json appBundle)
+        [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"
+
        arg_tls_cert=$(echo "$2" | $json tlsCert)
        arg_tls_key=$(echo "$2" | $json tlsKey)
        arg_token=$(echo "$2" | $json token)
@@ -4,4 +4,4 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connection=50
+max_connections=50
@@ -25,12 +25,12 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh

-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

 Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh

+Defaults!/home/yellowtent/box/src/scripts/setup_infra.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setup_infra.sh
+
@@ -1,5 +1,5 @@
 [Unit]
-Description=Cloudron Smart Cloud
+Description=Cloudron Smartserver
 Documentation=https://cloudron.io/documentation.html
 StopWhenUnneeded=true
 Requires=box.service
@@ -7,7 +7,7 @@ StopWhenUnneeded=false
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
-ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+ExecStart="/home/yellowtent/box/crashnotifierservice.js" %I
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 KillMode=process
 User=yellowtent
@@ -9,8 +9,6 @@ readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
 readonly ADMIN_LOCATION="my" # keep this in sync with constants.js

-source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
-
 echo "Setting up nginx update page"

 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
@@ -24,13 +22,16 @@ rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"

 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+readonly current_infra=$(node -e "console.log(require('${script_dir}/../src/infra_version.js').version);")
+existing_infra="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && existing_infra=$(node -e "console.log(JSON.parse(require('fs').readFileSync('${DATA_DIR}/INFRA_VERSION', 'utf8')).version);")
+if [[ "${arg_retire}" == "true" || "${existing_infra}" != "${current_infra}" ]]; then
+    echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire} existing: ${existing_infra} current: ${current_infra}"
    rm -f ${DATA_DIR}/nginx/applications/*
    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
+    echo "Show progress bar only on admin domain for normal update"
    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi
@@ -39,7 +39,7 @@ set_progress "10" "Ensuring directories"
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
 mkdir -p "${DATA_DIR}/box/appicons"
 mkdir -p "${DATA_DIR}/box/certs"
-mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/box/mail/dkim/${arg_fqdn}"
 mkdir -p "${DATA_DIR}/box/acme" # acme keys
 mkdir -p "${DATA_DIR}/graphite"

@@ -120,11 +120,9 @@ fi

 set_progress "33" "Changing ownership"
 chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"

-set_progress "40" "Setting up infra"
-${script_dir}/start/setup_infra.sh "${arg_fqdn}"
-
 set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
@@ -138,7 +136,6 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
    "fqdn": "${arg_fqdn}",
    "isCustomDomain": ${arg_is_custom_domain},
    "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "admin@${arg_fqdn}",
    "provider": "${arg_provider}",
    "database": {
        "hostname": "localhost",
@@ -146,7 +143,8 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
        "password": "${mysql_root_password}",
        "port": 3306,
        "name": "box"
-    }
+    },
+    "appBundle": ${arg_app_bundle}
 }
 CONF_END

@@ -194,12 +192,11 @@ fi
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
 echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+readonly ADMIN_SCOPES="cloudron,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box

 echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box

@@ -18,9 +18,10 @@ server {
    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
    # https://cipherli.st/
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
    ssl_prefer_server_ciphers on;
    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";

    proxy_http_version 1.1;
@@ -58,6 +59,7 @@ server {
            client_max_body_size 1m;
        }

+        # the read timeout is between successive reads and not the whole connection
        location ~ ^/api/v1/apps/.*/exec$ {
            proxy_pass   http://127.0.0.1:3000;
            proxy_read_timeout 30m;
@@ -78,6 +80,14 @@ server {
        proxy_pass http://127.0.0.1:3003;
        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
 <% } else if ( endpoint === 'app' ) { %>
+
+        # http://serverfault.com/questions/121766/webdav-rename-fails-on-an-apache-mod-dav-install-behind-nginx
+        set $destination $http_destination;
+        if ($destination ~* ^https(.+)$) {
+            set $destination http$1;
+        }
+        proxy_set_header Destination $destination;
+
        proxy_pass http://127.0.0.1:<%= port %>;
 <% } else if ( endpoint === 'splash' ) { %>
        root   <%= sourceDir %>;
@@ -24,7 +24,14 @@ http {

    sendfile        on;

-    keepalive_timeout  65;
+    # timeout for client to finish sending headers
+    client_header_timeout 30s;
+
+    # timeout for reading client request body (successive read timeout and not whole body!)
+    client_body_timeout 60s;
+
+    # keep-alive connections timeout in 65s. this is because many browsers timeout in 60 seconds
+    keepalive_timeout  65s;

    # HTTP server
    server {
@@ -50,22 +57,15 @@ http {
        }
    }

-    # We have to enable https for nginx to read in the vhost in http request
-    # and send a 404. This is a side-effect of using wildcard DNS
+    # This server handles the naked domain for custom domains.
+    # It can also be used for wildcard subdomain 404. This feature is not used by the Cloudron itself
+    # because box always sets up DNS records for app subdomains.
    server {
        listen 443 default_server;
        ssl on;
        ssl_certificate      cert/host.cert;
        ssl_certificate_key  cert/host.key;

-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # Disable check to allow unlimited body sizes
-        client_max_body_size 0;
-
        error_page 404 = @fallback;
        location @fallback {
            internal;
@@ -79,6 +79,7 @@ http {
            rewrite ^/$ /nakeddomain.html break;
        }

+        # required for /api/v1/cloudron/avatar
        location /api/ {
            proxy_pass http://127.0.0.1:3000;
            client_max_body_size 1m;
@@ -1,129 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-arg_fqdn="$1"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"
@@ -19,30 +19,34 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
    assert = require('assert'),
    async = require('async'),
-    child_process = require('child_process'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:addons'),
-    docker = require('./docker.js').connection,
+    docker = require('./docker.js'),
+    dockerConnection = docker.connection,
    fs = require('fs'),
    generatePassword = require('password-generator'),
    hat = require('hat'),
-    MemoryStream = require('memorystream'),
+    infra = require('./infra_version.js'),
    once = require('once'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    shell = require('./shell.js'),
-    spawn = child_process.spawn,
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');

 var NOOP = function (app, options, callback) { return callback(); };

 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    email: {
+        setup: setupEmail,
+        teardown: teardownEmail,
+        backup: NOOP,
+        restore: setupEmail
+    },
    ldap: {
        setup: setupLdap,
        teardown: teardownLdap,
@@ -79,6 +83,12 @@ var KNOWN_ADDONS = {
        backup: backupPostgreSql,
        restore: restorePostgreSql
    },
+    recvmail: {
+        setup: setupRecvMail,
+        teardown: teardownRecvMail,
+        backup: NOOP,
+        restore: setupRecvMail
+    },
    redis: {
        setup: setupRedis,
        teardown: teardownRedis,
@@ -207,17 +217,23 @@ function getLinksSync(app, addons) {

    if (!addons) return links;

+    var addMail = false;
+
    for (var addon in addons) {
        switch (addon) {
        case 'mysql': links.push('mysql:mysql'); break;
        case 'postgresql': links.push('postgresql:postgresql'); break;
-        case 'sendmail': links.push('mail:mail'); break;
+        case 'sendmail': addMail = true; break;
+        case 'recvmail': addMail = true; break;
+        case 'email': addMail = true; break;
        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
        case 'mongodb': links.push('mongodb:mongodb'); break;
        default: break;
        }
    }

+    if (addMail) links.push('mail:mail');
+
    return links;
 }

@@ -267,22 +283,18 @@ function setupOauth(app, options, callback) {
    assert.strictEqual(typeof callback, 'function');

    var appId = app.id;
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
    var redirectURI = 'https://' + config.appFqdn(app.location);
    var scope = 'profile';

-    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
-
-    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
+    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);

-        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
            if (error) return callback(error);

            var env = [
-                'OAUTH_CLIENT_ID=' + id,
-                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_CLIENT_ID=' + result.id,
+                'OAUTH_CLIENT_SECRET=' + result.clientSecret,
                'OAUTH_ORIGIN=' + config.adminOrigin()
            ];

@@ -300,7 +312,7 @@ function teardownOauth(app, options, callback) {

    debugApp(app, 'teardownOauth');

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);

        appdb.unsetAddonConfig(app.id, 'oauth', callback);
@@ -313,15 +325,12 @@ function setupSimpleAuth(app, options, callback) {
    assert.strictEqual(typeof callback, 'function');

    var appId = app.id;
-    var id = 'cid-' + uuid.v4();
    var scope = 'profile';

-    debugApp(app, 'setupSimpleAuth: id:%s', id);
-
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);

-        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+        clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) {
            if (error) return callback(error);

            var env = [
@@ -329,7 +338,7 @@ function setupSimpleAuth(app, options, callback) {
                'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_CLIENT_ID=' + id
+                'SIMPLE_AUTH_CLIENT_ID=' + result.id
            ];

            debugApp(app, 'Setting simple auth addon config to %j', env);
@@ -346,13 +355,44 @@ function teardownSimpleAuth(app, options, callback) {

    debugApp(app, 'teardownSimpleAuth');

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) {
        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);

        appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
    });
 }

+function setupEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2525',
+        'MAIL_IMAP_SERVER=mail',
+        'MAIL_IMAP_PORT=9993',
+        'MAIL_SIEVE_SERVER=mail',
+        'MAIL_SIEVE_PORT=4190',
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up Email');
+
+    appdb.setAddonConfig(app.id, 'email', env, callback);
+}
+
+function teardownEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down Email');
+
+    appdb.unsetAddonConfig(app.id, 'email', callback);
+}
+
 function setupLdap(app, options, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof options, 'object');
@@ -388,18 +428,17 @@ function setupSendMail(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var username = app.location ? app.location + '-app' : 'no-reply'; // use no-reply for bare domains
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';

-    var env = [
-        'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + username,
-        'MAIL_DOMAIN=' + config.fqdn()
-    ];
+    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];

-    debugApp(app, 'Setting up sendmail');
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);

-    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting sendmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    });
 }

 function teardownSendMail(app, options, callback) {
@@ -409,7 +448,55 @@ function teardownSendMail(app, options, callback) {

    debugApp(app, 'Tearing down sendmail');

-    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
+
+    debugApp(app, 'Tearing down sendmail');
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    });
+}
+
+function setupRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up recvmail');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
+
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting recvmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'recvmail', env, callback);
+    });
+}
+
+function teardownRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
+
+    debugApp(app, 'Tearing down recvmail');
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
+    });
 }

 function setupMySql(app, options, callback) {
@@ -419,31 +506,14 @@ function setupMySql(app, options, callback) {

    debugApp(app, 'Setting up mysql');

-    var container = docker.getContainer('mysql');
    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { bufferStdout: true }, function (error, stdout) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mysql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mysql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mysql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mysql', env, callback);
    });
 }

@@ -452,24 +522,14 @@ function teardownMySql(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = docker.getContainer('mysql');
    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];

    debugApp(app, 'Tearing down mysql');

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { }, function (error) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mysql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mysql', callback);
    });
 }

@@ -481,15 +541,9 @@ function backupMySql(app, options, callback) {
    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
    output.on('error', callback);

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ];

-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mysql', cmd, { stdout: output }, callback);
 }

 function restoreMySql(app, options, callback) {
@@ -503,17 +557,8 @@ function restoreMySql(app, options, callback) {
        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
        input.on('error', callback);

-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMySql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ];
+        docker.execContainer('mysql', cmd, { stdin: input }, callback);
    });
 }

@@ -524,31 +569,14 @@ function setupPostgreSql(app, options, callback) {

    debugApp(app, 'Setting up postgresql');

-    var container = docker.getContainer('postgresql');
    var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { bufferStdout: true }, function (error, stdout) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting postgresql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'postgresql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting postgresql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'postgresql', env, callback);
    });
 }

@@ -557,24 +585,14 @@ function teardownPostgreSql(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = docker.getContainer('postgresql');
    var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];

    debugApp(app, 'Tearing down postgresql');

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { }, function (error) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'postgresql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'postgresql', callback);
    });
 }

@@ -586,19 +604,13 @@ function backupPostgreSql(app, options, callback) {
    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
    output.on('error', callback);

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/postgresql/service.sh', 'backup', app.id ];

-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('postgresql', cmd, { stdout: output }, callback);
 }

 function restorePostgreSql(app, options, callback) {
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    callback = once(callback);

    setupPostgreSql(app, options, function (error) {
        if (error) return callback(error);
@@ -608,17 +620,9 @@ function restorePostgreSql(app, options, callback) {
        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
        input.on('error', callback);

-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
-        });
+        var cmd = [ '/addons/postgresql/service.sh', 'restore', app.id ];

-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        docker.execContainer('postgresql', cmd, { stdin: input }, callback);
    });
 }

@@ -629,31 +633,14 @@ function setupMongoDb(app, options, callback) {

    debugApp(app, 'Setting up mongodb');

-    var container = docker.getContainer('mongodb');
    var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { bufferStdout: true }, function (error, stdout) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mongodb addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mongodb addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mongodb', env, callback);
    });
 }

@@ -662,24 +649,14 @@ function teardownMongoDb(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var container = docker.getContainer('mongodb');
    var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];

    debugApp(app, 'Tearing down mongodb');

-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { }, function (error) {
        if (error) return callback(error);

-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mongodb', callback);
    });
 }

@@ -691,15 +668,9 @@ function backupMongoDb(app, options, callback) {
    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
    output.on('error', callback);

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mongodb/service.sh', 'backup', app.id ];

-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mongodb', cmd, { stdout: output }, callback);
 }

 function restoreMongoDb(app, options, callback) {
@@ -713,26 +684,16 @@ function restoreMongoDb(app, options, callback) {
        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
        input.on('error', callback);

-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mongodb/service.sh', 'restore', app.id ];
+        docker.execContainer('mongodb', cmd, { stdin: input }, callback);
    });
 }

-
 function forwardRedisPort(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    docker.getContainer('redis-' + appId).inspect(function (error, data) {
+    dockerConnection.getContainer('redis-' + appId).inspect(function (error, data) {
        if (error) return callback(new Error('Unable to inspect container:' + error));

        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
@@ -780,7 +741,7 @@ function setupRedis(app, options, callback) {
        name: 'redis-' + app.id,
        Hostname: 'redis-' + app.location,
        Tty: true,
-        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
+        Image: infra.images.redis.tag,
        Cmd: null,
        Volumes: {
            '/tmp': {},
@@ -812,9 +773,9 @@ function setupRedis(app, options, callback) {
        'REDIS_PORT=6379'
    ];

-    var redisContainer = docker.getContainer(createOptions.name);
+    var redisContainer = dockerConnection.getContainer(createOptions.name);
    stopAndRemoveRedis(redisContainer, function () {
-        docker.createContainer(createOptions, function (error) {
+        dockerConnection.createContainer(createOptions, function (error) {
            if (error && error.statusCode !== 409) return callback(error); // if not already created

            redisContainer.start(function (error) {
@@ -835,7 +796,7 @@ function teardownRedis(app, options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

-   var container = docker.getContainer('redis-' + app.id);
+   var container = dockerConnection.getContainer('redis-' + app.id);

   var removeOptions = {
       force: true, // kill container if it's running
@@ -858,15 +819,7 @@ function teardownRedis(app, options, callback) {
 function backupRedis(app, options, callback) {
    debugApp(app, 'Backing up redis');

-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    var cmd = [ '/addons/redis/service.sh', 'backup' ]; // the redis dir is volume mounted

-    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
-    });
-
-    cp.stdout.pipe(process.stdout);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('redis-' + app.id, cmd, { }, callback);
 }
@@ -4,7 +4,6 @@

 exports = module.exports = {
    get: get,
-    getBySubdomain: getBySubdomain,
    getByHttpPort: getByHttpPort,
    getByContainerId: getByContainerId,
    add: add,
@@ -59,7 +58,7 @@ var assert = require('assert'),

 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain' ].join(',');

 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');

@@ -114,22 +113,6 @@ function get(id, callback) {
    });
 }

-function getBySubdomain(subdomain, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
 function getByHttpPort(httpPort, callback) {
    assert.strictEqual(typeof httpPort, 'number');
    assert.strictEqual(typeof callback, 'function');
@@ -177,7 +160,7 @@ function getAll(callback) {
    });
 }

-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, callback) {
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, altDomain, callback) {
    assert.strictEqual(typeof id, 'string');
    assert.strictEqual(typeof appStoreId, 'string');
    assert(manifest && typeof manifest === 'object');
@@ -186,6 +169,7 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
    assert.strictEqual(typeof portBindings, 'object');
    assert.strictEqual(typeof accessRestriction, 'object');
    assert.strictEqual(typeof memoryLimit, 'number');
+    assert(altDomain === null || typeof altDomain === 'string');
    assert.strictEqual(typeof callback, 'function');

    portBindings = portBindings || { };
@@ -195,8 +179,8 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction

    var queries = [ ];
    queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit, altDomain ]
    });

    Object.keys(portBindings).forEach(function (env) {
@@ -26,9 +26,10 @@ function debugApp(app) {
    assert(!app || typeof app === 'object');

    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
+    var manifestAppId = app ? app.manifest.id : '';
    var id = app ? app.id : '';

-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + id);
+    debug(prefix + ' ' + manifestAppId + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + id);
 }

 function setHealth(app, health, callback) {
@@ -118,7 +119,7 @@ function processApps(callback) {

            var alive = apps
                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
-                .map(function (a) { return a.location || 'naked_domain'; }).join(', ');
+                .map(function (a) { return (a.location || 'naked_domain') + '|' + a.manifest.id; }).join(', ');

            debug('apps alive: [%s]', alive);

@@ -161,7 +162,7 @@ function processDockerEvents() {
                debug('OOM Context: %s', context);

                // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+                if (error || app.appStoreId !== '') mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
            });
        });

@@ -1,15 +1,11 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
    AppsError: AppsError,

    hasAccessTo: hasAccessTo,
-    requiresOAuthProxy: requiresOAuthProxy,

    get: get,
-    getBySubdomain: getBySubdomain,
    getByIpAddress: getByIpAddress,
    getAll: getAll,
    getAllByUser: getAllByUser,
@@ -19,12 +15,10 @@ exports = module.exports = {
    uninstall: uninstall,

    restore: restore,
-    restoreApp: restoreApp,

    update: update,

    backup: backup,
-    backupApp: backupApp,
    listBackups: listBackups,

    getLogs: getLogs,
@@ -36,9 +30,10 @@ exports = module.exports = {

    checkManifestConstraints: checkManifestConstraints,

-    setRestorePoint: setRestorePoint,
+    updateApps: updateApps,

-    autoupdateApps: autoupdateApps,
+    restoreInstalledApps: restoreInstalledApps,
+    configureInstalledApps: configureInstalledApps,

    // exported for testing
    _validateHostname: validateHostname,
@@ -51,13 +46,13 @@ var addons = require('./addons.js'),
    assert = require('assert'),
    async = require('async'),
    backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
    certificates = require('./certificates.js'),
    config = require('./config.js'),
    constants = require('./constants.js'),
    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:apps'),
    docker = require('./docker.js'),
+    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    groups = require('./groups.js'),
    manifestFormat = require('cloudron-manifestformat'),
@@ -65,34 +60,14 @@ var addons = require('./addons.js'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    semver = require('semver'),
-    shell = require('./shell.js'),
    spawn = require('child_process').spawn,
    split = require('split'),
    superagent = require('superagent'),
    taskmanager = require('./taskmanager.js'),
    util = require('util'),
+    uuid = require('node-uuid'),
    validator = require('validator');

-var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
-    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
-
-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 // http://dustinsenos.com/articles/customErrorsInNode
 // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
 function AppsError(reason, errorOrMessage) {
@@ -132,16 +107,16 @@ AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name
 function validateHostname(location, fqdn) {
-    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION ];
+    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, constants.MAIL_LOCATION, constants.POSTMAN_LOCATION ];

-    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new Error(location + ' is reserved');
+    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new AppsError(AppsError.BAD_FIELD, location + ' is reserved');

    if (location === '') return null; // bare location

-    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new Error('Hostname length cannot be greater than 63');
-    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new Error('Hostname can only contain alphanumerics and hyphen');
-    if (location[0] === '-' || location[location.length-1] === '-') return new Error('Hostname cannot start or end with hyphen');
-    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new Error('FQDN length exceeds 253 characters');
+    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new AppsError(AppsError.BAD_FIELD, 'Hostname length cannot be greater than 63');
+    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new AppsError(AppsError.BAD_FIELD, 'Hostname can only contain alphanumerics and hyphen');
+    if (location[0] === '-' || location[location.length-1] === '-') return new AppsError(AppsError.BAD_FIELD, 'Hostname cannot start or end with hyphen');
+    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new AppsError(AppsError.BAD_FIELD, 'FQDN length exceeds 253 characters');

    return null;
 }
@@ -155,17 +130,23 @@ function validatePortBindings(portBindings, tcpPorts) {
        25, /* smtp */
        53, /* dns */
        80, /* http */
+        143, /* imap */
        443, /* https */
+        465, /* smtps */
+        587, /* submission */
        919, /* ssh */
+        993, /* imaps */
        2003, /* graphite (lo) */
        2004, /* graphite (lo) */
        2020, /* install server */
        config.get('port'), /* app server (lo) */
-        config.get('internalPort'), /* internal app server (lo) */
+        config.get('sysadminPort'), /* sysadmin app server (lo) */
+        config.get('smtpPort'), /* internal smtp port (lo) */
        config.get('ldapPort'), /* ldap server (lo) */
        config.get('oauthProxyPort'), /* oauth proxy server (lo) */
        config.get('simpleAuthPort'), /* simple auth server (lo) */
        3306, /* mysql (lo) */
+        4190, /* managesieve */
        8000 /* graphite (lo) */
    ];

@@ -175,8 +156,8 @@ function validatePortBindings(portBindings, tcpPorts) {
    for (env in portBindings) {
        if (!/^[a-zA-Z0-9_]+$/.test(env)) return new AppsError(AppsError.BAD_FIELD, env + ' is not valid environment variable');

-        if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
-        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
+        if (!Number.isInteger(portBindings[env])) return new AppsError(AppsError.BAD_FIELD, portBindings[env] + ' is not an integer');
+        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new AppsError(AppsError.BAD_FIELD, portBindings[env] + ' is out of range');

        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, String(portBindings[env]));
    }
@@ -199,19 +180,20 @@ function validateAccessRestriction(accessRestriction) {
    var noUsers = true, noGroups = true;

    if (accessRestriction.users) {
-        if (!Array.isArray(accessRestriction.users)) return new Error('users array property required');
-        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+        if (!Array.isArray(accessRestriction.users)) return new AppsError(AppsError.BAD_FIELD, 'users array property required');
+        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All users have to be strings');
        noUsers = accessRestriction.users.length === 0;
    }

    if (accessRestriction.groups) {
-        if (!Array.isArray(accessRestriction.groups)) return new Error('groups array property required');
-        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new Error('All groups have to be strings');
+        if (!Array.isArray(accessRestriction.groups)) return new AppsError(AppsError.BAD_FIELD, 'groups array property required');
+        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All groups have to be strings');
        noGroups = accessRestriction.groups.length === 0;
    }

-    if (noUsers && noGroups) return new Error('users and groups array cannot both be empty');
+    if (noUsers && noGroups) return new AppsError(AppsError.BAD_FIELD, 'users and groups array cannot both be empty');

+    // TODO: maybe validate if the users and groups actually exist
    return null;
 }

@@ -226,8 +208,8 @@ function validateMemoryLimit(manifest, memoryLimit) {
    // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
    if (memoryLimit === 0) return null;

-    if (memoryLimit < min) return new Error('memoryLimit too small');
-    if (memoryLimit > max) return new Error('memoryLimit too large');
+    if (memoryLimit < min) return new AppsError(AppsError.BAD_FIELD, 'memoryLimit too small');
+    if (memoryLimit > max) return new AppsError(AppsError.BAD_FIELD, 'memoryLimit too large');

    return null;
 }
@@ -281,19 +263,6 @@ function hasAccessTo(app, user, callback) {
    });
 }

-function requiresOAuthProxy(app) {
-    assert.strictEqual(typeof app, 'object');
-
-    var tmp = app.accessRestriction;
-
-    // if no accessRestriction set, or the app uses one of the auth modules, we do not need the oauth proxy
-    if (tmp === null) return false;
-    if (app.manifest.addons['ldap'] || app.manifest.addons['oauth'] || app.manifest.addons['simpleauth']) return false;
-
-    // check if any restrictions are set
-    return !!((tmp.users && tmp.users.length) || (tmp.groups && tmp.groups.length));
-}
-
 function get(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');
@@ -303,22 +272,7 @@ function get(appId, callback) {
        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

        app.iconUrl = getIconUrlSync(app);
-        app.fqdn = config.appFqdn(app.location);
-
-        callback(null, app);
-    });
-}
-
-function getBySubdomain(subdomain, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    appdb.getBySubdomain(subdomain, function (error, app) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        app.iconUrl = getIconUrlSync(app);
-        app.fqdn = config.appFqdn(app.location);
+        app.fqdn = app.altDomain || config.appFqdn(app.location);

        callback(null, app);
    });
@@ -336,7 +290,7 @@ function getByIpAddress(ip, callback) {
            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

            app.iconUrl = getIconUrlSync(app);
-            app.fqdn = config.appFqdn(app.location);
+            app.fqdn = app.altDomain || config.appFqdn(app.location);

            callback(null, app);
        });
@@ -351,7 +305,7 @@ function getAll(callback) {

        apps.forEach(function (app) {
            app.iconUrl = getIconUrlSync(app);
-            app.fqdn = config.appFqdn(app.location);
+            app.fqdn = app.altDomain || config.appFqdn(app.location);
        });

        callback(null, apps);
@@ -395,198 +349,269 @@ function purchase(appStoreId, callback) {
    });
 }

-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert(!icon || typeof icon === 'string');
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof memoryLimit, 'number');
+function downloadManifest(appStoreId, manifest, callback) {
+    if (!appStoreId && !manifest) return callback(new AppsError(AppsError.BAD_FIELD, 'Neither manifest nor appStoreId provided'));
+
+    if (!appStoreId) return callback(null, '', manifest);
+
+    var parts = appStoreId.split('@');
+
+    var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
+
+    debug('downloading manifest from %s', url);
+
+    superagent.get(url).end(function (error, result) {
+        if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Network error downloading manifest:' + error.message));
+
+        if (result.statusCode !== 200) return callback(new AppsError(AppsError.BAD_FIELD, util.format('Failed to get app info from store.', result.statusCode, result.text)));
+
+        callback(null, parts[0], result.body.manifest);
+    });
+}
+
+function install(data, auditSource, callback) {
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var error = manifestFormat.parse(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
+    var location = data.location.toLowerCase(),
+        portBindings = data.portBindings || null,
+        accessRestriction = data.accessRestriction || null,
+        icon = data.icon || null,
+        cert = data.cert || null,
+        key = data.key || null,
+        memoryLimit = data.memoryLimit || 0,
+        altDomain = data.altDomain || null;

-    error = checkManifestConstraints(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+    assert(data.appStoreId || data.manifest); // atleast one of them is required

-    error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validatePortBindings(portBindings, manifest.tcpPorts);
-    if (error) return callback(error);
-
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validateMemoryLimit(manifest, memoryLimit);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    // memoryLimit might come in as 0 if not specified
-    memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-
-    // singleUser mode requires accessRestriction to contain exactly one user
-    if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
-    if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
-
-    if (icon) {
-        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
-
-        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
-            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
-        }
-    }
-
-    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
-    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
-
-    debug('Will install app with id : ' + appId);
-
-    purchase(appStoreId, function (error) {
+    downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
        if (error) return callback(error);

-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        error = manifestFormat.parse(manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));

-            // save cert to data/box/certs
-            if (cert && key) {
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        error = checkManifestConstraints(manifest);
+        if (error) return callback(error);
+
+        error = validateHostname(location, config.fqdn());
+        if (error) return callback(error);
+
+        error = validatePortBindings(portBindings, manifest.tcpPorts);
+        if (error) return callback(error);
+
+        error = validateAccessRestriction(accessRestriction);
+        if (error) return callback(error);
+
+        error = validateMemoryLimit(manifest, memoryLimit);
+        if (error) return callback(error);
+
+        // memoryLimit might come in as 0 if not specified
+        memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
+        if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+
+        // singleUser mode requires accessRestriction to contain exactly one user
+        if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
+        if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
+
+        if (icon) {
+            if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+
+            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+                return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
            }
+        }

-            taskmanager.restartAppTask(appId);
+        error = certificates.validateCertificate(cert, key, config.appFqdn(location));
+        if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));

-            callback(null);
+        var appId = uuid.v4();
+        debug('Will install app with id : ' + appId);
+
+        purchase(appStoreId, function (error) {
+            if (error) return callback(error);
+
+            appdb.add(appId, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, altDomain, function (error) {
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                // save cert to data/box/certs
+                if (cert && key) {
+                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+                }
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+
+                callback(null, { id : appId });
+            });
        });
    });
 }

-function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, callback) {
+function configure(appId, data, auditSource, callback) {
    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof memoryLimit, 'number');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
-    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
-
    appdb.get(appId, function (error, app) {
        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

-        error = validatePortBindings(portBindings, app.manifest.tcpPorts);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-        error = validateMemoryLimit(app.manifest, memoryLimit);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-        // memoryLimit might come in as 0 if not specified
-        memoryLimit = memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-
-        // save cert to data/box/certs
-        if (cert && key) {
-            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        var location, portBindings, values = { };
+        if ('location' in data) {
+            location = values.location = data.location.toLowerCase();
+            error = validateHostname(values.location, config.fqdn());
+            if (error) return callback(error);
+        } else {
+            location = app.location;
        }

-        var values = {
-            location: location.toLowerCase(),
-            accessRestriction: accessRestriction,
-            portBindings: portBindings,
-            memoryLimit: memoryLimit,
+        if ('accessRestriction' in data) {
+            values.accessRestriction = data.accessRestriction;
+            error = validateAccessRestriction(values.accessRestriction);
+            if (error) return callback(error);
+        }

-            oldConfig: {
-                location: app.location,
-                accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings,
-                memoryLimit: app.memoryLimit
+        if ('altDomain' in data) {
+            values.altDomain = data.altDomain;
+            if (values.altDomain !== null && !validator.isFQDN(values.altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+        }
+
+        if ('portBindings' in data) {
+            portBindings = values.portBindings = data.portBindings;
+            error = validatePortBindings(values.portBindings, app.manifest.tcpPorts);
+            if (error) return callback(error);
+        } else {
+            portBindings = app.portBindings;
+        }
+
+        if ('memoryLimit' in data) {
+            values.memoryLimit = data.memoryLimit;
+            error = validateMemoryLimit(app.manifest, values.memoryLimit);
+            if (error) return callback(error);
+
+            // memoryLimit might come in as 0 if not specified
+            values.memoryLimit = values.memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+        }
+
+        // save cert to data/box/certs. TODO: move this to apptask when we have a real task queue
+        if ('cert' in data && 'key' in data) {
+            if (data.cert && data.key) {
+                error = certificates.validateCertificate(data.cert, data.key, config.appFqdn(location));
+                if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), data.cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), data.key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+            } else { // remove existing cert/key
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'))) debug('Error removing cert: ' + safe.error.message);
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'))) debug('Error removing key: ' + safe.error.message);
            }
+        }
+
+        values.oldConfig = {
+            location: app.location,
+            accessRestriction: app.accessRestriction,
+            portBindings: app.portBindings,
+            memoryLimit: app.memoryLimit,
+            altDomain: app.altDomain
        };

        debug('Will configure app with id:%s values:%j', appId, values);

        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_CONFIGURE, values, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

            taskmanager.restartAppTask(appId);

+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId });
+
            callback(null);
        });
    });
 }

-function update(appId, force, manifest, portBindings, icon, callback) {
+function update(appId, data, auditSource, callback) {
    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof force, 'boolean');
-    assert(manifest && typeof manifest === 'object');
-    assert(!portBindings || typeof portBindings === 'object');
-    assert(!icon || typeof icon === 'string');
+    assert(data && typeof data === 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug('Will update app with id:%s', appId);

-    var error = manifestFormat.parse(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
+    downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
+        if (error) return callback(error);

-    error = checkManifestConstraints(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed:' + error.message));
+        var values = { };

-    error = validatePortBindings(portBindings, manifest.tcpPorts);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+        error = manifestFormat.parse(manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));

-    if (icon) {
-        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+        error = checkManifestConstraints(manifest);
+        if (error) return callback(error);

-        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
-            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+        values.manifest = manifest;
+
+        if ('portBindings' in data) {
+            values.portBindings = data.portBindings;
+            error = validatePortBindings(data.portBindings, values.manifest.tcpPorts);
+            if (error) return callback(error);
        }
-    }

-    appdb.get(appId, function (error, app) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if ('icon' in data) {
+            if (data.icon) {
+                if (!validator.isBase64(data.icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));

-        // Ensure we update the memory limit in case the new app requires more memory as a minimum
-        var memoryLimit = manifest.memoryLimit ? (app.memoryLimit < manifest.memoryLimit ? manifest.memoryLimit : app.memoryLimit) : app.memoryLimit;
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(data.icon, 'base64'))) {
+                    return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+                }
+            } else {
+                safe.fs.unlinkSync(path.join(paths.APPICONS_DIR, appId + '.png'));
+            }
+        }

-        var values = {
-            manifest: manifest,
-            portBindings: portBindings,
-            memoryLimit: memoryLimit,
+        appdb.get(appId, function (error, app) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

-            oldConfig: {
+            // prevent user from installing a app with different manifest id over an existing app
+            // this allows cloudron install -f --app <appid> for an app installed from the appStore
+            if (app.manifest.id !== values.manifest.id) {
+                if (!data.force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
+                // clear appStoreId so that this app does not get updates anymore. this will mark it as a dev app
+                values.appStoreId = '';
+            }
+
+            // Ensure we update the memory limit in case the new app requires more memory as a minimum
+            if (values.manifest.memoryLimit && app.memoryLimit < values.manifest.memoryLimit) {
+                values.memoryLimit = values.manifest.memoryLimit;
+            }
+
+            values.oldConfig = {
                manifest: app.manifest,
                portBindings: app.portBindings,
                accessRestriction: app.accessRestriction,
-                memoryLimit: app.memoryLimit
-            }
-        };
+                memoryLimit: app.memoryLimit,
+                altDomain: app.altDomain
+            };

-        appdb.setInstallationCommand(appId, force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, portBindings, error));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+            appdb.setInstallationCommand(appId, data.force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, values.portBindings, error));
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

-            taskmanager.restartAppTask(appId);
+                taskmanager.restartAppTask(appId);

-            callback(null);
+                eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest });
+
+                callback(null);
+            });
        });
    });
 }
@@ -636,8 +661,9 @@ function getLogs(appId, lines, follow, callback) {
    });
 }

-function restore(appId, callback) {
+function restore(appId, auditSource, callback) {
    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug('Will restore app with id:%s', appId);
@@ -652,7 +678,7 @@ function restore(appId, callback) {
            // re-validate because this new box version may not accept old configs.
            // if we restore location, it should be validated here as well
            error = checkManifestConstraints(restoreConfig.manifest);
-            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+            if (error) return callback(error);

            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
            if (error) return callback(error);
@@ -668,7 +694,8 @@ function restore(appId, callback) {
                    accessRestriction: app.accessRestriction,
                    portBindings: app.portBindings,
                    memoryLimit: app.memoryLimit,
-                    manifest: app.manifest
+                    manifest: app.manifest,
+                    altDomain: app.altDomain
                }
            };
        }
@@ -679,13 +706,16 @@ function restore(appId, callback) {

            taskmanager.restartAppTask(appId);

+            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { appId: appId });
+
            callback(null);
        });
    });
 }

-function uninstall(appId, callback) {
+function uninstall(appId, auditSource, callback) {
    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug('Will uninstall app with id:%s', appId);
@@ -695,6 +725,8 @@ function uninstall(appId, callback) {
            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

+            eventlog.add(eventlog.ACTION_APP_UNINSTALL, auditSource, { appId: appId });
+
            taskmanager.startAppTask(appId, callback);
        });
    });
@@ -733,14 +765,14 @@ function stop(appId, callback) {
 }

 function checkManifestConstraints(manifest) {
-    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+    if (!manifest.dockerImage) return new AppsError(AppsError.BAD_FIELD, 'Missing dockerImage'); // dockerImage is optional in manifest

    if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
-        return new Error('Box version exceeds Apps maxBoxVersion');
+        return new AppsError(AppsError.BAD_FIELD, 'Box version exceeds Apps maxBoxVersion');
    }

    if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, config.version())) {
-        return new Error('minBoxVersion exceeds Box version');
+        return new AppsError(AppsError.BAD_FIELD, 'minBoxVersion exceeds Box version');
    }

    return null;
@@ -764,10 +796,14 @@ function exec(appId, options, callback) {

        var container = docker.connection.getContainer(app.containerId);

-       var execOptions = {
+        var execOptions = {
            AttachStdin: true,
            AttachStdout: true,
            AttachStderr: true,
+            // A pseudo tty is a terminal which processes can detect (for example, disable colored output)
+            // Creating a pseudo terminal also assigns a terminal driver which detects control sequences
+            // When passing binary data, tty must be disabled. In addition, the stdout/stderr becomes a single
+            // unified stream because of the nature of a tty (see https://github.com/docker/docker/issues/19696)
            Tty: options.tty,
            Cmd: cmd
        };
@@ -777,9 +813,18 @@ function exec(appId, options, callback) {
            var startOptions = {
                Detach: false,
                Tty: options.tty,
-                stdin: true // this is a dockerode option that enabled openStdin in the modem
+                // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
+                // we can work with half-close connections (not defined in http). this way, the client
+                // can properly signal that stdin is EOF by closing it's side of the socket. In http,
+                // the whole connection will be dropped when stdin get EOF.
+                // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
+                hijack: true,
+                stream: true,
+                stdin: true,
+                stdout: true,
+                stderr: true
            };
-            exec.start(startOptions, function(error, stream) {
+            exec.start(startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

                if (options.rows && options.columns) {
@@ -792,37 +837,25 @@ function exec(appId, options, callback) {
    });
 }

-function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof lastBackupId, 'string');
-    assert.strictEqual(typeof lastBackupConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        return callback(null);
-    });
-}
-
-function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
+function updateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    function canAutoupdateApp(app, newManifest) {
-        var tcpPorts = newManifest.tcpPorts || { };
+        var newTcpPorts = newManifest.newTcpPorts || { };
+        var oldTcpPorts = app.manifest.tcpPorts || { };
        var portBindings = app.portBindings; // this is never null

-        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
-        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
-        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
-
-        for (var env in tcpPorts) {
-            if (!(env in portBindings)) return new Error(env + ' is required from user');
+        for (var env in newTcpPorts) {
+            if (!(env in oldTcpPorts)) return new Error(env + ' is required from user');
        }

-        // it's fine if one or more keys got removed
+        for (env in portBindings) {
+            if (!(env in newTcpPorts)) return new Error(env + ' was in use but new update removes it');
+        }
+
+        // it's fine if one or more (unused) keys got removed
        return null;
    }

@@ -841,7 +874,11 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
                return iteratorDone();
            }

-           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+            var data = {
+                manifest: updateInfo[appId].manifest
+            };
+
+            update(appId, data, auditSource, function (error) {
                if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);

                iteratorDone(null);
@@ -850,102 +887,6 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
    }, callback);
 }

-function canBackupApp(app) {
-    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
-    // state not good for consistent backup (i.e addons may not have been setup completely)
-    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
-            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
-            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
-            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
-}
-
-// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
-// s3 does not allow changing creation time, so copying the last backup is easy way out for now
-function reuseOldBackup(app, callback) {
-    assert.strictEqual(typeof app.lastBackupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    backups.copyLastBackup(app, function (error, newBackupId) {
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
-
-        callback(null, newBackupId);
-    });
-}
-
-function createNewBackup(app, addonsToBackup, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    backups.getBackupUrl(app, function (error, backupArchive) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        backups.getAppBackupConfigUrl(app, function (error, backupConfig) {
-            if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            debugApp(app, 'backupApp: backup url:%s backup config url:%s', backupArchive.url, backupConfig.url);
-
-            async.series([
-                ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-                addons.backupAddons.bind(null, app, addonsToBackup),
-                shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, backupArchive.url, backupConfig.url, backupArchive.backupKey, backupArchive.sessionToken ]),
-                ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-            ], function (error) {
-                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-                callback(null, backupArchive.id);
-            });
-        });
-    });
-}
-
-function backupApp(app, addonsToBackup, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var appConfig = null, backupFunction;
-
-    if (!canBackupApp(app)) {
-        if (!app.lastBackupId) {
-            debugApp(app, 'backupApp: cannot backup app');
-            return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy and never backed up previously'));
-        }
-
-        appConfig = app.lastBackupConfig;
-        backupFunction = reuseOldBackup.bind(null, app);
-    } else {
-        appConfig = {
-            manifest: app.manifest,
-            location: app.location,
-            portBindings: app.portBindings,
-            accessRestriction: app.accessRestriction,
-            memoryLimit: app.memoryLimit
-        };
-        backupFunction = createNewBackup.bind(null, app, addonsToBackup);
-
-        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
-            return callback(safe.error);
-        }
-    }
-
-    backupFunction(function (error, backupId) {
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        debugApp(app, 'backupApp: successful id:%s', backupId);
-
-        setRestorePoint(app.id, backupId, appConfig, function (error) {
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            return callback(null, backupId);
-        });
-    });
-}
-
 function backup(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');
@@ -965,28 +906,11 @@ function backup(appId, callback) {
    });
 }

-function restoreApp(app, addonsToRestore, backupId, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToRestore, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-    assert(app.lastBackupId);

-    backups.getRestoreUrl(backupId, function (error, result) {
-        if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+function listBackups(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);

-        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
-
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            addons.restoreAddons(app, addonsToRestore, callback);
-        });
-    });
-}
-
-function listBackups(appId, callback) {
    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

@@ -994,22 +918,42 @@ function listBackups(appId, callback) {
        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
        if (!exists) return callback(new AppsError(AppsError.NOT_FOUND));

-        // TODO pagination is not implemented in the backend yet
-        backups.getAllPaged(0, 1000, function (error, result) {
+        backups.getByAppIdPaged(page, perPage, appId, function (error, results) {
            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

-            var appBackups = [];
-
-            result.forEach(function (backup) {
-                appBackups = appBackups.concat(backup.dependsOn.filter(function (d) {
-                    return d.indexOf('appbackup_' + appId) === 0;
-                }));
-            });
-
-            // alphabetic should be sufficient
-            appBackups.sort();
-
-            callback(null, appBackups);
+            callback(null, results);
        });
    });
 }
+
+function restoreInstalledApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        async.map(apps, function (app, iteratorDone) {
+            if (app.installationState !== appdb.ISTATE_INSTALLED) return iteratorDone();
+
+            debug('marking %s for restore', app.location || app.id);
+
+            appdb.setInstallationCommand(app.id, appdb.ISTATE_PENDING_RESTORE, { oldConfig: null }, iteratorDone);
+        }, callback);
+    });
+}
+
+function configureInstalledApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        async.map(apps, function (app, iteratorDone) {
+            if (app.installationState !== appdb.ISTATE_INSTALLED) return iteratorDone();
+
+            debug('marking %s for reconfigure', app.location || app.id);
+
+            appdb.setInstallationCommand(app.id, appdb.ISTATE_PENDING_CONFIGURE, { oldConfig: null }, iteratorDone);
+        }, callback);
+    });
+}
@@ -1,7 +1,5 @@
 #!/usr/bin/env node

-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -19,7 +17,8 @@ exports = module.exports = {
    _verifyManifest: verifyManifest,
    _registerSubdomain: registerSubdomain,
    _unregisterSubdomain: unregisterSubdomain,
-    _waitForDnsPropagation: waitForDnsPropagation
+    _waitForDnsPropagation: waitForDnsPropagation,
+    _waitForAltDomainDnsPropagation: waitForAltDomainDnsPropagation
 };

 require('supererror')({ splatchError: true });
@@ -35,8 +34,9 @@ var addons = require('./addons.js'),
    apps = require('./apps.js'),
    assert = require('assert'),
    async = require('async'),
+    backups = require('./backups.js'),
    certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
    database = require('./database.js'),
    DatabaseError = require('./databaseerror.js'),
@@ -44,7 +44,6 @@ var addons = require('./addons.js'),
    docker = require('./docker.js'),
    ejs = require('ejs'),
    fs = require('fs'),
-    hat = require('hat'),
    manifestFormat = require('cloudron-manifestformat'),
    net = require('net'),
    nginx = require('./nginx.js'),
@@ -57,7 +56,7 @@ var addons = require('./addons.js'),
    superagent = require('superagent'),
    sysinfo = require('./sysinfo.js'),
    util = require('util'),
-    uuid = require('node-uuid'),
+    waitForDns = require('./waitfordns.js'),
    _ = require('underscore');

 var COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
@@ -100,13 +99,10 @@ function configureNginx(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var vhost = config.appFqdn(app.location);
-    var oauthProxy = apps.requiresOAuthProxy(app);
-
-    certificates.ensureCertificate(vhost, function (error, certFilePath, keyFilePath) {
+    certificates.ensureCertificate(app, function (error, certFilePath, keyFilePath) {
        if (error) return callback(error);

-        nginx.configureApp(app, oauthProxy, certFilePath, keyFilePath, callback);
+        nginx.configureApp(app, certFilePath, keyFilePath, callback);
    });
 }

@@ -163,21 +159,19 @@ function allocateOAuthProxyCredentials(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    if (!apps.requiresOAuthProxy(app)) return callback(null);
+    if (!nginx.requiresOAuthProxy(app)) return callback(null);

-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
    var redirectURI = 'https://' + config.appFqdn(app.location);
    var scope = 'profile';

-    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
 }

 function removeOAuthProxyCredentials(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
        if (error && error.reason !== DatabaseError.NOT_FOUND) {
            debugApp(app, 'Error removing OAuth client id', error);
            return callback(error);
@@ -232,17 +226,19 @@ function downloadIcon(app, callback) {

    var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';

-    superagent
-        .get(iconUrl)
-        .buffer(true)
-        .end(function (error, res) {
-            if (error && !error.response) return callback(new Error('Network error downloading icon:' + error.message));
-            if (res.statusCode !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        superagent
+            .get(iconUrl)
+            .buffer(true)
+            .end(function (error, res) {
+                if (error && !error.response) return retryCallback(new Error('Network error downloading icon:' + error.message));
+                if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli

-            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return retryCallback(new Error('Error saving icon:' + safe.error.message));

-            callback(null);
-    });
+                retryCallback(null);
+        });
+    }, callback);
 }

 function registerSubdomain(app, callback) {
@@ -319,20 +315,23 @@ function waitForDnsPropagation(app, callback) {
        return callback(null);
    }

-    function retry(error) {
-        debugApp(app, 'waitForDnsPropagation: ', error);
-        setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
-    }
+    async.retry({ interval: 5000, times: 120 }, function checkStatus(retryCallback) {
+        subdomains.status(app.dnsRecordId, function (error, result) {
+            if (error) return retryCallback(new Error('Failed to get dns record status : ' + error.message));

-    subdomains.status(app.dnsRecordId, function (error, result) {
-        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);

-        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
+            if (result !== 'done') return retryCallback(new Error(util.format('app:%s not ready yet: %s', app.id, result)));

-        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
+            retryCallback(null, result);
+        });
+    }, callback);
+}

-        callback(null);
-    });
+function waitForAltDomainDnsPropagation(app, callback) {
+    if (!app.altDomain) return callback(null);
+
+    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', callback); // waits forever
 }

 // updates the app object and the database
@@ -411,9 +410,12 @@ function install(app, callback) {

        runApp.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
        exports._waitForDnsPropagation.bind(null, app),

+        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
+
        updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
        configureNginx.bind(null, app),

@@ -437,7 +439,7 @@ function backup(app, callback) {

    async.series([
        updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        apps.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest.addons),

        // done!
        function (callback) {
@@ -502,7 +504,7 @@ function restore(app, callback) {
        createVolume.bind(null, app),

        updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons, backupId),
+        backups.restoreApp.bind(null, app, app.manifest.addons, backupId),

        updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
        createContainer.bind(null, app),
@@ -512,9 +514,12 @@ function restore(app, callback) {

        runApp.bind(null, app),

-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
        exports._waitForDnsPropagation.bind(null, app),

+        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
        updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
        configureNginx.bind(null, app),

@@ -574,6 +579,9 @@ function configure(app, callback) {
        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
        exports._waitForDnsPropagation.bind(null, app),

+        updateApp.bind(null, app, { installationProgress: '85, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
        updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
        configureNginx.bind(null, app),

@@ -630,7 +638,7 @@ function update(app, callback) {

            async.series([
                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
-                apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                backups.backupApp.bind(null, app, app.oldConfig.manifest.addons)
            ], next);
        },

@@ -10,7 +10,7 @@ exports = module.exports = {
 var assert = require('assert'),
    BasicStrategy = require('passport-http').BasicStrategy,
    BearerStrategy = require('passport-http-bearer').Strategy,
-    clientdb = require('./clientdb'),
+    clients = require('./clients'),
    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
    DatabaseError = require('./databaseerror'),
    debug = require('debug')('box:auth'),
@@ -28,11 +28,11 @@ function initialize(callback) {
    assert.strictEqual(typeof callback, 'function');

    passport.serializeUser(function (user, callback) {
-        callback(null, user.username);
+        callback(null, user.id);
    });

-    passport.deserializeUser(function(username, callback) {
-        userdb.get(username, function (error, result) {
+    passport.deserializeUser(function(userId, callback) {
+        userdb.get(userId, function (error, result) {
            if (error) return callback(error);

            var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -44,7 +44,7 @@ function initialize(callback) {

    passport.use(new LocalStrategy(function (username, password, callback) {
        if (username.indexOf('@') === -1) {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                if (error) return callback(error);
@@ -67,14 +67,14 @@ function initialize(callback) {
            debug('BasicStrategy: detected client id %s instead of username:password', username);
            // username is actually client id here
            // password is client secret
-            clientdb.get(username, function (error, client) {
+            clients.get(username, function (error, client) {
                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                if (error) return callback(error);
                if (client.clientSecret != password) return callback(null, false);
                return callback(null, client);
            });
        } else {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                if (error) return callback(error);
@@ -85,7 +85,7 @@ function initialize(callback) {
    }));

    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
+        clients.get(clientId, function(error, client) {
            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
            if (error) { return callback(error); }
            if (client.clientSecret != clientSecret) { return callback(null, false); }
@@ -101,29 +101,11 @@ function initialize(callback) {
            // scopes here can define what capabilities that token carries
            // passport put the 'info' object into req.authInfo, where we can further validate the scopes
            var info = { scope: token.scope };
-            var tokenType;
-
-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }

            userdb.get(token.identifier, function (error, user) {
                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                if (error) return callback(error);

-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
                // amend the admin flag
                groups.isMember(groups.ADMIN_GROUP_ID, user.id, function (error, isAdmin) {
                    if (error) return callback(error);
@@ -0,0 +1,114 @@
+'use strict';
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', ];
+
+exports = module.exports = {
+    add: add,
+    getPaged: getPaged,
+    get: get,
+    del: del,
+    getByAppIdPaged: getByAppIdPaged,
+
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+}
+
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, 'appbackup\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
+        [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function add(backup, callback) {
+    assert(backup && typeof backup === 'object');
+    assert.strictEqual(typeof backup.id, 'string');
+    assert.strictEqual(typeof backup.version, 'string');
+    assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
+    assert(util.isArray(backup.dependsOn));
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = backup.creationTime || new Date(); // allow tests to set the time
+
+    database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn) VALUES (?, ?, ?, ?, ?, ?)',
+        [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(',') ],
+        function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
@@ -3,22 +3,54 @@
 exports = module.exports = {
    BackupsError: BackupsError,

-    getAllPaged: getAllPaged,
+    getPaged: getPaged,
+    getByAppIdPaged: getByAppIdPaged,

-    getBackupUrl: getBackupUrl,
-    getAppBackupConfigUrl: getAppBackupConfigUrl,
    getRestoreUrl: getRestoreUrl,

-    copyLastBackup: copyLastBackup
+    ensureBackup: ensureBackup,
+
+    backup: backup,
+    backupApp: backupApp,
+    restoreApp: restoreApp,
+
+    backupBoxAndApps: backupBoxAndApps
 };

-var assert = require('assert'),
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    async = require('async'),
+    assert = require('assert'),
+    backupdb = require('./backupdb.js'),
    caas = require('./storage/caas.js'),
    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:backups'),
+    eventlog = require('./eventlog.js'),
+    locker = require('./locker.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    progress = require('./progress.js'),
    s3 = require('./storage/s3.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
    settings = require('./settings.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
+
+var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}

 function BackupsError(reason, errorOrMessage) {
    assert.strictEqual(typeof reason, 'string');
@@ -41,6 +73,7 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.BAD_STATE = 'bad state';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';

 // choose which storage backend we use for test purpose we use s3
@@ -52,74 +85,78 @@ function api(provider) {
    }
 }

-function getAllPaged(page, perPage, callback) {
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
    assert.strictEqual(typeof callback, 'function');

-    settings.getBackupConfig(function (error, backupConfig) {
+    backupdb.getPaged(page, perPage, function (error, results) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
-            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-
-            return callback(null, backups); // [ { creationTime, restoreKey } ] sorted by time (latest first
-        });
+        callback(null, results);
    });
 }

-function getBackupUrl(app, callback) {
-    assert(!app || typeof app === 'object');
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var filename = '';
-    if (app) {
-        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    } else {
-        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
-    }
+    backupdb.getByAppIdPaged(page, perPage, appId, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getBoxBackupCredentials(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
+    var filename = filebase + '.tar.gz';

    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
            if (error) return callback(error);

-            var obj = {
-                id: filename,
-                url: result.url,
-                sessionToken: result.sessionToken,
-                backupKey: backupConfig.key
-            };
+            result.id = filename;
+            result.s3Url = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + filename;
+            result.backupKey = backupConfig.key;

-            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getBoxBackupCredentials: %j', result);

-            callback(null, obj);
+            callback(null, result);
        });
    });
 }

-function getAppBackupConfigUrl(app, callback) {
+function getAppBackupCredentials(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

-    var filename = util.format('appbackup_%s_%s-v%s.json', app.id, (new Date()).toISOString(), app.manifest.version);
+    var now = new Date();
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
+    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';

    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
            if (error) return callback(error);

-            var obj = {
-                id: filename,
-                url: result.url,
-                sessionToken: result.sessionToken
-            };
+            result.id = dataFilename;
+            result.s3ConfigUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + configFilename;
+            result.s3DataUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + dataFilename;
+            result.backupKey = backupConfig.key;

-            debug('getAppBackupConfigUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getAppBackupCredentials: %j', result);

-            callback(null, obj);
+            callback(null, result);
        });
    });
 }
@@ -132,17 +169,16 @@ function getRestoreUrl(backupId, callback) {
    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

-        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
+        api(backupConfig.provider).getRestoreUrl(backupConfig, backupId, function (error, result) {
            if (error) return callback(error);

            var obj = {
                id: backupId,
                url: result.url,
-                sessionToken: result.sessionToken,
                backupKey: backupConfig.key
            };

-            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getRestoreUrl: id:%s url:%s backupKey:%s', obj.id, obj.url, obj.backupKey);

            callback(null, obj);
        });
@@ -154,18 +190,23 @@ function copyLastBackup(app, callback) {
    assert.strictEqual(typeof app.lastBackupId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, (new Date()).toISOString(), app.manifest.version);
+    var now = new Date();
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), app.manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), app.manifest.version);

    settings.getBackupConfig(function (error, backupConfig) {
        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));

+        debug('copyLastBackup: copying archive %s to %s', app.lastBackupId, toFilenameArchive);
+
        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));

            // TODO change that logic by adjusting app.lastBackupId to not contain the file type
            var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';

+            debug('copyLastBackup: copying config %s to %s', configFileId, toFilenameConfig);
+
            api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
                if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));

@@ -174,3 +215,258 @@ function copyLastBackup(app, callback) {
        });
    });
 }
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    getBoxBackupCredentials(appBackupIds, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        debug('backupBoxWithAppBackupIds:  %j', result);
+
+        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.sessionToken, result.region, result.backupKey ];
+
+        shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debug('backupBoxWithAppBackupIds: success');
+
+            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                    if (error) return callback(error);
+                    callback(null, result.id);
+                });
+            });
+        });
+    });
+}
+
+// this function expects you to have a lock
+// function backupBox(callback) {
+//    apps.getAll(function (error, allApps) {
+//         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+//
+//         var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+//         appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+//
+//         backupBoxWithAppBackupIds(appBackupIds, callback);
+//     });
+// }
+
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldAppBackup(app, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    copyLastBackup(app, function (error, newBackupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewAppBackup(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAppBackupCredentials(app, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
+
+        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey,
+                     result.sessionToken, result.region, result.backupKey ];
+
+        async.series([
+            addons.backupAddons.bind(null, app, addonsToBackup),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
+        ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debugApp(app, 'createNewAppBackup: %s done', result.id);
+
+            backupdb.add({ id: result.id, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, result.id);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof lastBackupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function backupApp(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appConfig = null, backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        appConfig = app.lastBackupConfig;
+        backupFunction = reuseOldAppBackup.bind(null, app);
+    } else {
+        appConfig = {
+            manifest: app.manifest,
+            location: app.location,
+            portBindings: app.portBindings,
+            accessRestriction: app.accessRestriction,
+            memoryLimit: app.memoryLimit
+        };
+        backupFunction = createNewAppBackup.bind(null, app, addonsToBackup);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, appConfig, function (error) {
+            if (error) return callback(error);
+
+            return callback(null, backupId);
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    callback = callback || NOOP_CALLBACK;
+
+    eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { });
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            backupApp(app, app.manifest.addons, function (error, backupId) {
+                if (error && error.reason !== BackupsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
+                }
+
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+
+                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, filename: filename });
+
+                callback(error, filename);
+            });
+        });
+    });
+}
+
+function backup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
+
+    progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
+
+    backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background
+        if (error) console.error('backup failed.', error);
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    getPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        backup(auditSource, callback);
+    });
+}
+
+function restoreApp(app, addonsToRestore, backupId, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    getRestoreUrl(backupId, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
+        });
+    });
+}
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 var assert = require('assert'),
@@ -7,6 +5,7 @@ var assert = require('assert'),
    crypto = require('crypto'),
    debug = require('debug')('box:cert/acme'),
    fs = require('fs'),
+    parseLinks = require('parse-links'),
    path = require('path'),
    paths = require('../paths.js'),
    safe = require('safetydance'),
@@ -58,7 +57,6 @@ function Acme(options) {
    this.caOrigin = options.prod ? CA_PROD : CA_STAGING;
    this.accountKeyPem = null; // Buffer
    this.email = options.email;
-    this.chainPem = options.prod ? safe.fs.readFileSync(__dirname + '/lets-encrypt-x1-cross-signed.pem.txt') : new Buffer('');
 }

 Acme.prototype.getNonce = function (callback) {
@@ -304,7 +302,7 @@ Acme.prototype.signCertificate = function (domain, csrDer, callback) {

        if (!certUrl) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Missing location in downloadCertificate'));

-        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // for renewal
+        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // maybe use for renewal

        return callback(null, result.headers.location);
    });
@@ -315,25 +313,57 @@ Acme.prototype.createKeyAndCsr = function (domain, callback) {
    assert.strictEqual(typeof callback, 'function');

    var outdir = paths.APP_CERTS_DIR;
+    var csrFile = path.join(outdir, domain + '.csr');
+    var privateKeyFile = path.join(outdir, domain + '.key');
    var execSync = safe.child_process.execSync;

-    var privateKeyFile = path.join(outdir, domain + '.key');
-    var key = execSync('openssl genrsa 4096');
-    if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (safe.fs.existsSync(privateKeyFile)) {
+        // in some old releases, csr file was corrupt. so always regenerate it
+        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
+    } else {
+        var key = execSync('openssl genrsa 4096');
+        if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));

-    debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+    }

    var csrDer = execSync(util.format('openssl req -new -key %s -outform DER -subj /CN=%s', privateKeyFile, domain));
    if (!csrDer) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    var csrFile = path.join(outdir, domain + '.csr');
-    if (!safe.fs.writeFileSync(csrFile, csrFile)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error)); // bookkeeping

    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);

    callback(null, csrDer);
 };

+// TODO: download the chain in a loop following 'up' header
+Acme.prototype.downloadChain = function (linkHeader, callback) {
+    if (!linkHeader) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Empty link header when downloading certificate chain');
+
+    var linkInfo = parseLinks(linkHeader);
+    if (!linkInfo || !linkInfo.up) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Failed to parse link header when downloading certificate chain'); 
+
+    debug('downloadChain: downloading from %s', this.caOrigin + linkInfo.up);
+
+    superagent.get(this.caOrigin + linkInfo.up).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var chainDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        var chainPem = execSync('openssl x509 -inform DER -outform PEM', { input: chainDer }); // this is really just base64 encoding with header
+        if (!chainPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        callback(null, chainPem);
+    });
+};
+
 Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof certUrl, 'string');
@@ -355,18 +385,22 @@ Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
        var execSync = safe.child_process.execSync;

        safe.fs.writeFileSync(path.join(outdir, domain + '.der'), certificateDer);
-        debug('downloadCertificate: cert der file saved');
+        debug('downloadCertificate: cert der file for %s saved', domain);

        var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { input: certificateDer }); // this is really just base64 encoding with header
        if (!certificatePem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));

-        var certificateFile = path.join(outdir, domain + '.cert');
-        var fullChainPem = Buffer.concat([certificatePem, that.chainPem]);
-        if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        that.downloadChain(result.header['link'], function (error, chainPem) {
+            if (error) return callback(error);

-        debug('downloadCertificate: cert file saved at %s', certificateFile);
+            var certificateFile = path.join(outdir, domain + '.cert');
+            var fullChainPem = Buffer.concat([certificatePem, chainPem]);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));

-        callback();
+            debug('downloadCertificate: cert file for %s saved at %s', domain, certificateFile);
+
+            callback();
+        });
    });
 };

@@ -414,21 +448,11 @@ Acme.prototype.getCertificate = function (domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var outdir = paths.APP_CERTS_DIR;
-    var certUrl = safe.fs.readFileSync(path.join(outdir, domain + '.url'), 'utf8');
-    var certificateGetter;
-
-    if (certUrl) {
-        debug('getCertificate: renewing existing cert for %s from %s', domain, certUrl);
-        certificateGetter = this.downloadCertificate.bind(this, domain, certUrl);
-    } else {
-        debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
-        certificateGetter = this.acmeFlow.bind(this, domain);
-    }
-
-    certificateGetter(function (error) {
+    debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
+    this.acmeFlow(domain, function (error) {
        if (error) return callback(error);

+        var outdir = paths.APP_CERTS_DIR;
        callback(null, path.join(outdir, domain + '.cert'), path.join(outdir, domain + '.key'));
    });
 };
@@ -1,27 +0,0 @@
-----BEGIN CERTIFICATE-----
-MIIEqDCCA5CgAwIBAgIRAJgT9HUT5XULQ+dDHpceRL0wDQYJKoZIhvcNAQELBQAw
-PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD
-Ew5EU1QgUm9vdCBDQSBYMzAeFw0xNTEwMTkyMjMzMzZaFw0yMDEwMTkyMjMzMzZa
-MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
-ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMTCCASIwDQYJKoZIhvcNAQEBBQAD
-ggEPADCCAQoCggEBAJzTDPBa5S5Ht3JdN4OzaGMw6tc1Jhkl4b2+NfFwki+3uEtB
-BaupnjUIWOyxKsRohwuj43Xk5vOnYnG6eYFgH9eRmp/z0HhncchpDpWRz/7mmelg
-PEjMfspNdxIknUcbWuu57B43ABycrHunBerOSuu9QeU2mLnL/W08lmjfIypCkAyG
-dGfIf6WauFJhFBM/ZemCh8vb+g5W9oaJ84U/l4avsNwa72sNlRZ9xCugZbKZBDZ1
-gGusSvMbkEl4L6KWTyogJSkExnTA0DHNjzE4lRa6qDO4Q/GxH8Mwf6J5MRM9LTb4
-4/zyM2q5OTHFr8SNDR1kFjOq+oQpttQLwNh9w5MCAwEAAaOCAZIwggGOMBIGA1Ud
-EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMH8GCCsGAQUFBwEBBHMwcTAy
-BggrBgEFBQcwAYYmaHR0cDovL2lzcmcudHJ1c3RpZC5vY3NwLmlkZW50cnVzdC5j
-b20wOwYIKwYBBQUHMAKGL2h0dHA6Ly9hcHBzLmlkZW50cnVzdC5jb20vcm9vdHMv
-ZHN0cm9vdGNheDMucDdjMB8GA1UdIwQYMBaAFMSnsaR7LHH62+FLkHX/xBVghYkQ
-MFQGA1UdIARNMEswCAYGZ4EMAQIBMD8GCysGAQQBgt8TAQEBMDAwLgYIKwYBBQUH
-AgEWImh0dHA6Ly9jcHMucm9vdC14MS5sZXRzZW5jcnlwdC5vcmcwPAYDVR0fBDUw
-MzAxoC+gLYYraHR0cDovL2NybC5pZGVudHJ1c3QuY29tL0RTVFJPT1RDQVgzQ1JM
-LmNybDATBgNVHR4EDDAKoQgwBoIELm1pbDAdBgNVHQ4EFgQUqEpqYwR93brm0Tm3
-pkVl7/Oo7KEwDQYJKoZIhvcNAQELBQADggEBANHIIkus7+MJiZZQsY14cCoBG1hd
-v0J20/FyWo5ppnfjL78S2k4s2GLRJ7iD9ZDKErndvbNFGcsW+9kKK/TnY21hp4Dd
-ITv8S9ZYQ7oaoqs7HwhEMY9sibED4aXw09xrJZTC9zK1uIfW6t5dHQjuOWv+HHoW
-ZnupyxpsEUlEaFb+/SCI4KCSBdAsYxAcsHYI5xxEI4LutHp6s3OT2FuO90WfdsIk
-6q78OMSdn875bNjdBYAqxUp2/LEIHfDBkLoQz0hFJmwAbYahqKaLn73PAAm1X2kj
-f1w8DdnkabOLGeOVcj9LQ+s67vBykx4anTjURkbqZslUEUsn2k5xeua2zUk=
-----END CERTIFICATE-----
@@ -1,8 +1,18 @@
-/* jslint node:true */
-
 'use strict';

+exports = module.exports = {
+    installAdminCertificate: installAdminCertificate,
+    renewAll: renewAll,
+    setFallbackCertificate: setFallbackCertificate,
+    setAdminCertificate: setAdminCertificate,
+    CertificatesError: CertificatesError,
+    validateCertificate: validateCertificate,
+    ensureCertificate: ensureCertificate,
+    getAdminCertificatePath: getAdminCertificatePath
+};
+
 var acme = require('./cert/acme.js'),
+    apps = require('./apps.js'),
    assert = require('assert'),
    async = require('async'),
    caas = require('./cert/caas.js'),
@@ -10,7 +20,9 @@ var acme = require('./cert/acme.js'),
    config = require('./config.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:src/certificates'),
+    eventlog = require('./eventlog.js'),
    fs = require('fs'),
+    mailer = require('./mailer.js'),
    nginx = require('./nginx.js'),
    path = require('path'),
    paths = require('./paths.js'),
@@ -22,16 +34,6 @@ var acme = require('./cert/acme.js'),
    waitForDns = require('./waitfordns.js'),
    x509 = require('x509');

-exports = module.exports = {
-    installAdminCertificate: installAdminCertificate,
-    autoRenew: autoRenew,
-    setFallbackCertificate: setFallbackCertificate,
-    setAdminCertificate: setAdminCertificate,
-    CertificatesError: CertificatesError,
-    validateCertificate: validateCertificate,
-    ensureCertificate: ensureCertificate
-};
-
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };

 function CertificatesError(reason, errorOrMessage) {
@@ -55,15 +57,20 @@ function CertificatesError(reason, errorOrMessage) {
 util.inherits(CertificatesError, Error);
 CertificatesError.INTERNAL_ERROR = 'Internal Error';
 CertificatesError.INVALID_CERT = 'Invalid certificate';
+CertificatesError.NOT_FOUND = 'Not Found';
+
+function getApi(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');

-function getApi(callback) {
    settings.getTlsConfig(function (error, tlsConfig) {
        if (error) return callback(error);

-        var api = tlsConfig.provider === 'caas' ? caas : acme;
+        var api = !app.altDomain && tlsConfig.provider === 'caas' ? caas : acme;

        var options = { };
-        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        // used by acme backend to determine the LE origin.
+        options.prod = (api === caas) ? !config.isDev() : tlsConfig.provider.match(/.*-prod/) !== null;

        // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
        // we cannot use admin@fqdn because the user might not have set it up.
@@ -88,10 +95,10 @@ function installAdminCertificate(callback) {
        sysinfo.getIp(function (error, ip) {
            if (error) return callback(error);

-            waitForDns(config.adminFqdn(), ip, config.fqdn(), function (error) {
+            waitForDns(config.adminFqdn(), ip, 'A', function (error) {
                if (error) return callback(error); // this cannot happen because we retry forever

-                ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
+                ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
                    if (error) { // currently, this can never happen
                        debug('Error obtaining certificate. Proceed anyway', error);
                        return callback();
@@ -104,41 +111,89 @@ function installAdminCertificate(callback) {
    });
 }

-function needsRenewalSync(certFilePath) {
-    var result = safe.child_process.execSync('openssl x509 -checkend %s -in %s', 60 * 60 * 24 * 5, certFilePath);
+function isExpiringSync(certFilePath, hours) {
+    assert.strictEqual(typeof certFilePath, 'string');
+    assert.strictEqual(typeof hours, 'number');

-    return result === null; // command errored
+    if (!fs.existsSync(certFilePath)) return 2; // not found
+
+    var result = safe.child_process.spawnSync('/usr/bin/openssl', [ 'x509', '-checkend', String(60 * 60 * hours), '-in', certFilePath ]);
+
+    debug('isExpiringSync: %s %s %s', certFilePath, result.stdout.toString('utf8').trim(), result.status);
+
+    return result.status === 1; // 1 - expired 0 - not expired
 }

-function autoRenew(callback) {
-    debug('autoRenew: Checking certificates for renewal');
-    callback = callback || NOOP_CALLBACK;
+function renewAll(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');

-    var filenames = safe.fs.readdirSync(paths.APP_CERTS_DIR);
-    if (!filenames) {
-        debug('autoRenew: Error getting filenames: %s', safe.error.message);
-        return;
-    }
+    debug('renewAll: Checking certificates for renewal');

-    var certs = filenames.filter(function (f) {
-        return f.match(/\.cert$/) !== null && needsRenewalSync(path.join(paths.APP_CERTS_DIR, f));
-    });
-
-    debug('autoRenew: %j needs to be renewed', certs);
-
-    getApi(function (error, api, apiOptions) {
+    apps.getAll(function (error, allApps) {
        if (error) return callback(error);

-        async.eachSeries(certs, function iterator(cert, iteratorCallback) {
-            var domain = cert.match(/^(.*)\.cert$/)[1];
-            if (domain === 'host') return iteratorCallback(); // cannot renew fallback cert
+        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app

-            debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+        var expiringApps = [ ];
+        for (var i = 0; i < allApps.length; i++) {
+            var appDomain = allApps[i].altDomain || config.appFqdn(allApps[i].location);
+            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
+            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');

-            api.getCertificate(domain, apiOptions, function (error) {
-                if (error) debug('autoRenew: could not renew cert for %s', domain, error);
+            if (!safe.fs.existsSync(keyFilePath)) {
+                debug('renewAll: no existing key file for %s. skipping', appDomain);
+                continue;
+            }

-                iteratorCallback(); // move on to next cert
+            if (isExpiringSync(certFilePath, 24 * 30)) { // expired or not found
+                expiringApps.push(allApps[i]);
+            }
+        }
+
+        debug('renewAll: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
+
+        async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+            var domain = app.altDomain || config.appFqdn(app.location);
+
+            getApi(app, function (error, api, apiOptions) {
+                if (error) return callback(error);
+
+                debug('renewAll: renewing cert for %s with options %j', domain, apiOptions);
+
+                api.getCertificate(domain, apiOptions, function (error) {
+                    var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+                    var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+                    var errorMessage = error ? error.message : '';
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
+                    mailer.certificateRenewed(domain, errorMessage);
+
+                    if (error) {
+                        debug('renewAll: could not renew cert for %s because %s', domain, error);
+
+                        // check if we should fallback if we expire in the coming day
+                        if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
+
+                        debug('renewAll: using fallback certs for %s since it expires soon', domain, error);
+
+                        certFilePath = 'cert/host.cert';
+                        keyFilePath = 'cert/host.key';
+                    } else {
+                        debug('renewAll: certificate for %s renewed', domain);
+                    }
+
+                    // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
+                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                        nginx.configureAdmin.bind(null, certFilePath, keyFilePath)
+                        : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
+
+                    configureFunc(function (ignoredError) {
+                        if (ignoredError) debug('fallbackExpiredCertificates: error reconfiguring app', ignoredError);
+
+                        iteratorCallback(); // move to next app
+                    });
+                });
            });
        });
    });
@@ -207,12 +262,20 @@ function setFallbackCertificate(cert, key, callback) {
    });
 }

+function getFallbackCertificatePath(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // any user fallback cert is always copied over to nginx cert dir
+    callback(null, path.join(paths.NGINX_CERT_DIR, 'host.cert'), path.join(paths.NGINX_CERT_DIR, 'host.key'));
+}
+
+// FIXME: setting admin cert needs to restart the mail container because it uses admin cert
 function setAdminCertificate(cert, key, callback) {
    assert.strictEqual(typeof cert, 'string');
    assert.strictEqual(typeof key, 'string');
    assert.strictEqual(typeof callback, 'function');

-    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
+    var vhost = config.adminFqdn();
    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');

@@ -226,10 +289,24 @@ function setAdminCertificate(cert, key, callback) {
    nginx.configureAdmin(certFilePath, keyFilePath, callback);
 }

-function ensureCertificate(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
+function getAdminCertificatePath(callback) {
    assert.strictEqual(typeof callback, 'function');

+    var vhost = config.adminFqdn();
+    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, certFilePath, keyFilePath);
+
+    getFallbackCertificatePath(callback);
+}
+
+function ensureCertificate(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var domain = app.altDomain || config.appFqdn(app.location);
+
    // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
    var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
    var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
@@ -237,12 +314,12 @@ function ensureCertificate(domain, callback) {
    if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
        debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);

-        if (!needsRenewalSync(userCertFilePath)) return callback(null, userCertFilePath, userKeyFilePath);
-
-        debug('ensureCertificate: %s cert require renewal', domain);
+        if (!isExpiringSync(userCertFilePath, 24 * 1)) return callback(null, userCertFilePath, userKeyFilePath);
    }

-    getApi(function (error, api, apiOptions) {
+    debug('ensureCertificate: %s cert require renewal', domain);
+
+    getApi(app, function (error, api, apiOptions) {
        if (error) return callback(error);

        debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);
@@ -14,13 +14,7 @@ exports = module.exports = {
    delByAppId: delByAppId,
    delByAppIdAndType: delByAppIdAndType,

-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    _clear: clear
 };

 var assert = require('assert'),
@@ -7,8 +7,29 @@ exports = module.exports = {
    get: get,
    del: del,
    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getByAppIdAndType: getByAppIdAndType,
    getClientTokensByUserId: getClientTokensByUserId,
-    delClientTokensByUserId: delClientTokensByUserId
+    delClientTokensByUserId: delClientTokensByUserId,
+    delByAppIdAndType: delByAppIdAndType,
+
+    // keep this in sync with start.sh ADMIN_SCOPES that generates the cid-webadmin
+    SCOPE_APPS: 'apps',
+    SCOPE_DEVELOPER: 'developer',
+    SCOPE_PROFILE: 'profile',
+    SCOPE_CLOUDRON: 'cloudron',
+    SCOPE_SETTINGS: 'settings',
+    SCOPE_USERS: 'users',
+
+    // roles are handled just like the above scopes, they are parallel to scopes
+    // scopes enclose API groups, roles specify the usage role
+    SCOPE_ROLE_SDK: 'roleSdk',
+
+    // client type enums
+    TYPE_EXTERNAL: 'external',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy',
+    TYPE_ADMIN: 'admin'
 };

 var assert = require('assert'),
@@ -47,10 +68,21 @@ ClientsError.INVALID_CLIENT = 'Invalid client';
 function validateScope(scope) {
    assert.strictEqual(typeof scope, 'string');

-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
+    var VALID_SCOPES = [
+        exports.SCOPE_APPS,
+        exports.SCOPE_DEVELOPER,
+        exports.SCOPE_PROFILE,
+        exports.SCOPE_CLOUDRON,
+        exports.SCOPE_SETTINGS,
+        exports.SCOPE_USERS,
+        '*',    // includes all scopes, but not roles
+        exports.SCOPE_ROLE_SDK
+    ];

-    // TODO maybe validate all individual scopes if they exist
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE, 'Empty scope not allowed');
+
+    var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE, 'Invalid scope. Available scopes are ' + VALID_SCOPES.join(', '));

    return null;
 }
@@ -108,13 +140,13 @@ function getAllWithDetailsByUserId(userId, callback) {
    assert.strictEqual(typeof userId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+    clientdb.getAllWithTokenCountByIdentifier(userId, function (error, results) {
        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
        if (error) return callback(error);

        var tmp = [];
        async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
+            if (record.type === exports.TYPE_ADMIN) {
                record.name = constants.ADMIN_NAME;
                record.location = constants.ADMIN_LOCATION;

@@ -129,10 +161,10 @@ function getAllWithDetailsByUserId(userId, callback) {
                    return callback(null);  // ignore error so we continue listing clients
                }

-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === exports.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
+                if (record.type === exports.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';

                record.location = result.location;

@@ -147,12 +179,23 @@ function getAllWithDetailsByUserId(userId, callback) {
    });
 }

+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getByAppIdAndType(appId, type, function (error, result) {
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
 function getClientTokensByUserId(clientId, userId, callback) {
    assert.strictEqual(typeof clientId, 'string');
    assert.strictEqual(typeof userId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
        if (error && error.reason === DatabaseError.NOT_FOUND) {
            // this can mean either that there are no tokens or the clientId is actually unknown
            clientdb.get(clientId, function (error/*, result*/) {
@@ -171,7 +214,7 @@ function delClientTokensByUserId(clientId, userId, callback) {
    assert.strictEqual(typeof userId, 'string');
    assert.strictEqual(typeof callback, 'function');

-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
        if (error && error.reason === DatabaseError.NOT_FOUND) {
            // this can mean either that there are no tokens or the clientId is actually unknown
            clientdb.get(clientId, function (error/*, result*/) {
@@ -184,3 +227,14 @@ function delClientTokensByUserId(clientId, userId, callback) {
        callback(null);
    });
 }
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(appId, type, function (error) {
+        if (error) return callback(error);
+        callback(null);
+    });
+}
@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
@@ -14,11 +12,8 @@ exports = module.exports = {
    sendHeartbeat: sendHeartbeat,

    updateToLatest: updateToLatest,
-    update: update,
    reboot: reboot,
-    backup: backup,
    retire: retire,
-    ensureBackup: ensureBackup,

    isConfiguredSync: isConfiguredSync,

@@ -27,19 +22,20 @@ exports = module.exports = {
    events: new (require('events').EventEmitter)(),

    EVENT_ACTIVATED: 'activated',
-    EVENT_CONFIGURED: 'configured'
+    EVENT_CONFIGURED: 'configured',
+    EVENT_FIRST_RUN: 'firstrun'
 };

 var apps = require('./apps.js'),
-    AppsError = require('./apps.js').AppsError,
    assert = require('assert'),
    async = require('async'),
    backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
+    constants = require('./constants.js'),
    debug = require('debug')('box:cloudron'),
    df = require('node-df'),
+    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    locker = require('./locker.js'),
    mailer = require('./mailer.js'),
@@ -58,12 +54,9 @@ var apps = require('./apps.js'),
    user = require('./user.js'),
    UserError = user.UserError,
    userdb = require('./userdb.js'),
-    util = require('util'),
-    webhooks = require('./webhooks.js');
+    util = require('util');

 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
    RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');

@@ -71,24 +64,9 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };

 var gUpdatingDns = false,                // flag for dns update reentrancy
    gCloudronDetails = null,             // cached cloudron details like region,size...
+    gAppstoreUserDetails = {},
    gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet

-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 function CloudronError(reason, errorOrMessage) {
    assert.strictEqual(typeof reason, 'string');
    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -112,10 +90,6 @@ CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
 CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
 CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
@@ -123,15 +97,36 @@ CloudronError.NOT_FOUND = 'Not found';
 function initialize(callback) {
    assert.strictEqual(typeof callback, 'function');

-    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    ensureDkimKeySync();

-    syncConfigState(callback);
+    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.on(exports.EVENT_FIRST_RUN, installAppBundle);
+
+    // check activation state for existing cloudrons that do not have first run file
+    // can be removed once cloudrons have been updated
+    isActivated(function (error, activated) {
+        if (error) return callback(error);
+
+        debug('initialize: cloudron %s activated', activated ? '' : 'not');
+
+        if (activated) fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+
+        if (!fs.existsSync(paths.FIRST_RUN_FILE)) {
+            // EE API is sync. do not keep the server waiting
+            debug('initialize: emitting first run event');
+            process.nextTick(function () { exports.events.emit(exports.EVENT_FIRST_RUN); });
+            fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+        }
+
+        syncConfigState(callback);
+    });
 }

 function uninitialize(callback) {
    assert.strictEqual(typeof callback, 'function');

    exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.removeListener(exports.EVENT_FIRST_RUN, installAppBundle);

    callback(null);
 }
@@ -140,6 +135,15 @@ function isConfiguredSync() {
    return gIsConfigured === true;
 }

+function isActivated(callback) {
+    user.getOwner(function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+        if (error) return callback(error);
+
+        callback(null, true);
+    });
+}
+
 function isConfigured(callback) {
    // set of rules to see if we have the configs required for cloudron to function
    // note this checks for missing configs and not invalid configs
@@ -184,55 +188,62 @@ function setTimeZone(ip, callback) {

    debug('setTimeZone ip:%s', ip);

-    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
+    // https://github.com/bluesmoon/node-geoip
+    // https://github.com/runk/node-maxmind
+    // { url: 'http://freegeoip.net/json/%s', jpath: 'time_zone' },
+    // { url: 'http://ip-api.com/json/%s', jpath: 'timezone' },
+    // { url: 'http://geoip.nekudo.com/api/%s', jpath: 'time_zone }
+
+    superagent.get('http://freegeoip.net/json/' + ip).end(function (error, result) {
        if ((error && !error.response) || result.statusCode !== 200) {
            debug('Failed to get geo location: %s', error.message);
            return callback(null);
        }

-        if (!result.body.timezone) {
+        if (!result.body.time_zone || typeof result.body.time_zone !== 'string') {
            debug('No timezone in geoip response : %j', result.body);
            return callback(null);
        }

-        debug('Setting timezone to ', result.body.timezone);
+        debug('Setting timezone to ', result.body.time_zone);

-        settings.setTimeZone(result.body.timezone, callback);
+        settings.setTimeZone(result.body.time_zone, callback);
    });
 }

-function activate(username, password, email, displayName, ip, callback) {
+function activate(username, password, email, displayName, ip, auditSource, callback) {
    assert.strictEqual(typeof username, 'string');
    assert.strictEqual(typeof password, 'string');
    assert.strictEqual(typeof email, 'string');
    assert.strictEqual(typeof displayName, 'string');
    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug('activating user:%s email:%s', username, email);

    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region

-    user.createOwner(username, password, email, displayName, function (error, userObject) {
+    user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
        if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error.reason === UserError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clients.getByAppIdAndType('webadmin', clients.TYPE_ADMIN, function (error, result) {
            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

            // Also generate a token so the admin creation can also act as a login
            var token = tokendb.generateToken();
            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day

-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

                // EE API is sync. do not keep the REST API reponse waiting
                process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });

+                eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
+
                callback(null, { token: token, expires: expires });
            });
        });
@@ -282,6 +293,7 @@ function getCloudronDetails(callback) {
            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));

            gCloudronDetails = result.body.box;
+            gAppstoreUserDetails = result.body.user;

            return callback(null, gCloudronDetails);
        });
@@ -323,6 +335,7 @@ function getConfig(callback) {
                        developerMode: developerMode,
                        region: result.region,
                        size: result.size,
+                        billing: !!gAppstoreUserDetails.billing,
                        memory: os.totalmem(),
                        provider: config.provider(),
                        cloudronName: cloudronName
@@ -344,6 +357,21 @@ function sendHeartbeat() {
    });
 }

+function ensureDkimKeySync() {
+    var dkimPrivateKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/private');
+    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
+
+    if (fs.existsSync(dkimPrivateKeyFile) && fs.existsSync(dkimPublicKeyFile)) {
+        debug('DKIM keys already present');
+        return;
+    }
+
+    debug('Generating new DKIM keys');
+
+    safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024');
+    safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM');
+}
+
 function readDkimPublicKeySync() {
    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
    var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
@@ -359,6 +387,7 @@ function readDkimPublicKeySync() {
    return publicKey;
 }

+// NOTE: if you change the SPF record here, be sure the wait check in mailer.js
 function txtRecordsWithSpf(callback) {
    assert.strictEqual(typeof callback, 'function');

@@ -372,16 +401,16 @@ function txtRecordsWithSpf(callback) {
        for (i = 0; i < txtRecords.length; i++) {
            if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF

-            validSpf = txtRecords[i].indexOf(' a:' + config.fqdn() + ' ') !== -1;
+            validSpf = txtRecords[i].indexOf(' a:' + config.adminFqdn() + ' ') !== -1;
            break;
        }

        if (validSpf) return callback(null, null);

        if (i == txtRecords.length) {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ~all"';
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ~all"';
        } else {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
        }

        return callback(null, txtRecords);
@@ -403,27 +432,33 @@ function addDnsRecords() {
    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';

    var dkimKey = readDkimPublicKeySync();
-    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
+    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));

    sysinfo.getIp(function (error, ip) {
        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));

-        var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
-        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
+        var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
        // t=s limits the domainkey to this domain and not it's subdomains
        var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
        // DMARC requires special setup if report email id is in different domain
        var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };

+        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };
+
        var records = [ ];
        if (config.isCustomDomain()) {
            records.push(webadminRecord);
            records.push(dkimRecord);
+            records.push(mxRecord);
        } else {
+            // for custom domains, we show a nakeddomain.html page
+            var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
+
            records.push(nakedDomainRecord);
            records.push(webadminRecord);
            records.push(dkimRecord);
            records.push(dmarcRecord);
+            records.push(mxRecord);
        }

        debug('addDnsRecords: %j', records);
@@ -459,8 +494,9 @@ function reboot(callback) {
    shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }

-function update(boxUpdateInfo, callback) {
+function update(boxUpdateInfo, auditSource, callback) {
    assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    if (!boxUpdateInfo) return callback(null);
@@ -468,6 +504,8 @@ function update(boxUpdateInfo, callback) {
    var error = locker.lock(locker.OP_BOX_UPDATE);
    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));

+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
    // ensure tools can 'wait' on progress
    progress.set(progress.UPDATE, 0, 'Starting');

@@ -499,13 +537,14 @@ function update(boxUpdateInfo, callback) {
 }


-function updateToLatest(callback) {
+function updateToLatest(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    var boxUpdateInfo = updateChecker.getUpdateInfo().box;
    if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));

-    update(boxUpdateInfo, callback);
+    update(boxUpdateInfo, auditSource, callback);
 }

 function doShortCircuitUpdate(boxUpdateInfo, callback) {
@@ -528,7 +567,7 @@ function doUpgrade(boxUpdateInfo, callback) {

    progress.set(progress.UPDATE, 5, 'Backing up for upgrade');

-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'upgrader' }, function (error) {
        if (error) return upgradeError(error);

        superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
@@ -557,7 +596,7 @@ function doUpdate(boxUpdateInfo, callback) {

    progress.set(progress.UPDATE, 5, 'Backing up for update');

-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
        if (error) return updateError(error);

        // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
@@ -604,119 +643,31 @@ function doUpdate(boxUpdateInfo, callback) {
    });
 }

-function backup(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_FULL_BACKUP);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    // ensure tools can 'wait' on progress
-    progress.set(progress.BACKUP, 0, 'Starting');
-
-    // start the backup operation in the background
-    backupBoxAndApps(function (error) {
-        if (error) console.error('backup failed.', error);
-
-        locker.unlock(locker.OP_FULL_BACKUP);
-    });
-
-    callback(null);
-}
-
-function ensureBackup(callback) {
+function installAppBundle(callback) {
    callback = callback || NOOP_CALLBACK;

-    backups.getAllPaged(1, 1, function (error, backups) {
-        if (error) {
-            debug('Unable to list backups', error);
-            return callback(error); // no point trying to backup if appstore is down
-        }
+    var bundle = config.get('appBundle');

-        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
-            debug('Previous backup was %j, no need to backup now', backups[0]);
-            return callback(null);
-        }
+    if (!bundle || bundle.length === 0) {
+        debug('installAppBundle: no bundle set');
+        return callback();
+    }

-        backup(callback);
-    });
-}
+    async.eachSeries(bundle, function (appInfo, iteratorCallback) {
+        debug('autoInstall: installing %s at %s', appInfo.appstoreId, appInfo.location);

-function backupBoxWithAppBackupIds(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
+        var data = {
+            appStoreId: appInfo.appstoreId,
+            location: appInfo.location,
+            portBindings: appInfo.portBindings || null,
+            accessRestriction: appInfo.accessRestriction || null,
+        };

-    backups.getBackupUrl(null /* app */, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+        apps.install(data, { userId: null, username: 'autoinstaller' }, iteratorCallback);
+    }, function (error) {
+        if (error) debug('autoInstallApps: ', error);

-        debug('backup: url %s', result.url);
-
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            debug('backup: successful');
-
-            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                if (error) return callback(error);
-                callback(null, result.id);
-            });
-        });
-    });
-}
-
-// this function expects you to have a lock
-function backupBox(callback) {
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
-
-        backupBoxWithAppBackupIds(appBackupIds, callback);
-    });
-}
-
-// this function expects you to have a lock
-function backupBoxAndApps(callback) {
-    callback = callback || NOOP_CALLBACK;
-
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var processed = 0;
-        var step = 100/(allApps.length+1);
-
-        progress.set(progress.BACKUP, processed, '');
-
-        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
-            ++processed;
-
-            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                if (error && error.reason !== AppsError.BAD_STATE) {
-                    debugApp(app, 'Unable to backup', error);
-                    return iteratorCallback(error);
-                }
-
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
-                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
-            });
-        }, function appsBackedUp(error, backupIds) {
-            if (error) {
-                progress.set(progress.BACKUP, 100, error.message);
-                return callback(error);
-            }
-
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
-
-            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
-                progress.set(progress.BACKUP, 100, error ? error.message : '');
-                callback(error, restoreKey);
-            });
-        });
+        callback();
    });
 }

@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.stat">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
    Instance "<%= appId %>-memory"
    Separator " \\n"
    <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
    </Result>
  </Table>

-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
    Instance "<%= appId %>-memory"
    Separator "\\n"
    <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
    </Result>
  </Table>

-  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker/<%= containerId %>/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
    Instance "<%= appId %>-cpu"
    Separator " \\n"
    <Result>
@@ -1,11 +1,7 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
    baseDir: baseDir,
-    dnsInSync: dnsInSync,
-    setDnsInSync: setDnsInSync,

    // values set here will be lost after a upgrade/update. use the sqlite database
    // for persistent values that need to be backed up
@@ -30,7 +26,9 @@ exports = module.exports = {
    // these values are derived
    adminOrigin: adminOrigin,
    internalAdminOrigin: internalAdminOrigin,
+    sysadminOrigin: sysadminOrigin, // caas routes
    adminFqdn: adminFqdn,
+    mailFqdn: mailFqdn,
    appFqdn: appFqdn,
    zoneName: zoneName,

@@ -58,14 +56,6 @@ function baseDir() {

 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');

-function dnsInSync() {
-    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
-}
-
-function setDnsInSync(content) {
-    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
-}
-
 function saveSync() {
    fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
@@ -83,16 +73,17 @@ function initConfig() {
    data.fqdn = 'localhost';

    data.token = null;
-    data.adminEmail = null;
    data.boxVersionsUrl = null;
    data.version = null;
    data.isCustomDomain = false;
    data.webServerOrigin = null;
-    data.internalPort = 3001;
+    data.smtpPort = 2525; // // this value comes from mail container
+    data.sysadminPort = 3001;
    data.ldapPort = 3002;
    data.oauthProxyPort = 3003;
    data.simpleAuthPort = 3004;
    data.provider = 'caas';
+    data.appBundle = [ ];

    if (exports.CLOUDRON) {
        data.port = 3000;
@@ -109,7 +100,6 @@ function initConfig() {
            name: 'boxtest'
        };
        data.token = 'APPSTORE_TOKEN';
-        data.adminEmail = 'test@cloudron.foo';
    } else {
        assert(false, 'Unknown environment. This should not happen!');
    }
@@ -172,6 +162,10 @@ function adminFqdn() {
    return appFqdn(constants.ADMIN_LOCATION);
 }

+function mailFqdn() {
+    return appFqdn(constants.MAIL_LOCATION);
+}
+
 function adminOrigin() {
    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
@@ -180,6 +174,10 @@ function internalAdminOrigin() {
    return 'http://127.0.0.1:' + get('port');
 }

+function sysadminOrigin() {
+    return 'http://127.0.0.1:' + get('sysadminPort');
+}
+
 function token() {
    return get('token');
 }
@@ -4,6 +4,11 @@
 exports = module.exports = {
    ADMIN_LOCATION: 'my',
    API_LOCATION: 'api', // this is unused but reserved for future use (#403)
+    SMTP_LOCATION: 'smtp',
+    IMAP_LOCATION: 'imap',
+    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
+    POSTMAN_LOCATION: 'postman', // used in dovecot bounces
+
    ADMIN_NAME: 'Settings',

    ADMIN_CLIENT_ID: 'webadmin', // oauth client id
@@ -7,6 +7,7 @@ exports = module.exports = {

 var apps = require('./apps.js'),
    assert = require('assert'),
+    backups = require('./backups.js'),
    certificates = require('./certificates.js'),
    cloudron = require('./cloudron.js'),
    config = require('./config.js'),
@@ -29,6 +30,7 @@ var gAutoupdaterJob = null,
    gCheckDiskSpaceJob = null;

 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var AUDIT_SOURCE = { userId: null, username: 'cron' };

 // cron format
 // Seconds: 0-59
@@ -65,7 +67,7 @@ function recreateJobs(unusedTimeZone, callback) {
        if (gBackupJob) gBackupJob.stop();
        gBackupJob = new CronJob({
            cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: cloudron.ensureBackup,
+            onTick: backups.ensureBackup.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
            start: true,
            timeZone: allSettings[settings.TIME_ZONE_KEY]
        });
@@ -121,7 +123,7 @@ function recreateJobs(unusedTimeZone, callback) {
        if (gCertificateRenewJob) gCertificateRenewJob.stop();
        gCertificateRenewJob = new CronJob({
            cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: certificates.autoRenew,
+            onTick: certificates.renewAll.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
            start: true,
            timeZone: allSettings[settings.TIME_ZONE_KEY]
        });
@@ -153,10 +155,10 @@ function autoupdatePatternChanged(pattern) {
            var updateInfo = updateChecker.getUpdateInfo();
            if (updateInfo.box) {
                debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
            } else if (updateInfo.apps) {
                debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+                apps.updateApps(updateInfo.apps, AUDIT_SOURCE, NOOP_CALLBACK);
            } else {
                debug('No auto updates available');
            }
@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
@@ -116,11 +114,14 @@ function clear(callback) {
    async.series([
        require('./appdb.js')._clear,
        require('./authcodedb.js')._clear,
+        require('./backupdb.js')._clear,
        require('./clientdb.js')._clear,
        require('./tokendb.js')._clear,
        require('./groupdb.js')._clear,
        require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear
+        require('./settingsdb.js')._clear,
+        require('./eventlogdb.js')._clear,
+        require('./mailboxdb.js')._clear
    ], callback);
 }

@@ -5,7 +5,7 @@
 exports = module.exports = {
    DeveloperError: DeveloperError,

-    enabled: enabled,
+    isEnabled: isEnabled,
    setEnabled: setEnabled,
    issueDeveloperToken: issueDeveloperToken,
    getNonApprovedApps: getNonApprovedApps
@@ -13,7 +13,9 @@ exports = module.exports = {

 var assert = require('assert'),
    config = require('./config.js'),
+    clients = require('./clients.js'),
    debug = require('debug')('box:developer'),
+    eventlog = require('./eventlog.js'),
    tokendb = require('./tokendb.js'),
    settings = require('./settings.js'),
    superagent = require('superagent'),
@@ -41,7 +43,7 @@ util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
 DeveloperError.EXTERNAL_ERROR = 'External Error';

-function enabled(callback) {
+function isEnabled(callback) {
    assert.strictEqual(typeof callback, 'function');

    settings.getDeveloperMode(function (error, enabled) {
@@ -50,27 +52,35 @@ function enabled(callback) {
    });
 }

-function setEnabled(enabled, callback) {
+function setEnabled(enabled, auditSource, callback) {
    assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    settings.setDeveloperMode(enabled, function (error) {
        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        eventlog.add(eventlog.ACTION_CLI_MODE, auditSource, { enabled: enabled });
+
        callback(null);
    });
 }

-function issueDeveloperToken(user, callback) {
+function issueDeveloperToken(user, auditSource, callback) {
    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    var token = tokendb.generateToken();
    var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var scopes = '*,' + clients.SCOPE_ROLE_SDK;

-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
+    tokendb.add(token, user.id, '', expiresAt, scopes, function (error) {
        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));

-        callback(null, { token: token, expiresAt: expiresAt });
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
+
+        callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
    });
 }

@@ -1,17 +1,5 @@
 'use strict';

-var addons = require('./addons.js'),
-    async = require('async'),
-    assert = require('assert'),
-    config = require('./config.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:src/docker.js'),
-    Docker = require('dockerode'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    util = require('util'),
-    _ = require('underscore');
-
 exports = module.exports = {
    connection: connectionInstance(),
    downloadImage: downloadImage,
@@ -25,10 +13,12 @@ exports = module.exports = {
    deleteImage: deleteImage,
    deleteContainers: deleteContainers,
    createSubcontainer: createSubcontainer,
-    getContainerIdByIp: getContainerIdByIp
+    getContainerIdByIp: getContainerIdByIp,
+    execContainer: execContainer
 };

 function connectionInstance() {
+    var Docker = require('dockerode');
    var docker;

    if (process.env.BOX_ENV === 'test') {
@@ -44,6 +34,20 @@ function connectionInstance() {
    return docker;
 }

+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    child_process = require('child_process'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/docker.js'),
+    once = require('once'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    spawn = child_process.spawn,
+    util = require('util'),
+    _ = require('underscore');
+
 function debugApp(app, args) {
    assert(!app || typeof app === 'object');

@@ -134,12 +138,13 @@ function createSubcontainer(app, name, cmd, options, callback) {
    var manifest = app.manifest;
    var developmentMode = !!manifest.developmentMode;
    var exposedPorts = {}, dockerPortBindings = { };
+    var domain = app.altDomain || config.appFqdn(app.location);
    var stdEnv = [
        'CLOUDRON=1',
        'WEBADMIN_ORIGIN=' + config.adminOrigin(),
        'API_ORIGIN=' + config.adminOrigin(),
-        'APP_ORIGIN=https://' + config.appFqdn(app.location),
-        'APP_DOMAIN=' + config.appFqdn(app.location)
+        'APP_ORIGIN=https://' + domain,
+        'APP_DOMAIN=' + domain
    ];

    // docker portBindings requires ports to be exposed
@@ -389,3 +394,34 @@ function getContainerIdByIp(ip, callback) {
        callback(null, containerId);
    });
 }
+
+function execContainer(containerId, cmd, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert(util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', '-i', containerId  ].concat(cmd));
+
+    var chunks = [ ];
+
+    if (options.stdout) {
+        cp.stdout.pipe(options.stdout);
+    } else if (options.bufferStdout) {
+        cp.stdout.on('data', function (chunk) { chunks.push(chunk); });
+    } else {
+        cp.stdout.pipe(process.stdout);
+    }
+
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debug('execContainer code: %s signal: %s', code, signal);
+        if (!callback.called) callback(code ? 'Failed with status ' + code : null, Buffer.concat(chunks));
+    });
+
+    cp.stderr.pipe(options.stderr || process.stderr);
+
+    if (options.stdin) options.stdin.pipe(cp.stdin).on('error', callback);
+}
@@ -0,0 +1,101 @@
+'use strict';
+
+exports = module.exports = {
+    EventLogError: EventLogError,
+
+    add: add,
+    get: get,
+    getAllPaged: getAllPaged,
+
+    // keep in sync with webadmin index.js filter
+    ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CONFIGURE: 'app.configure',
+    ACTION_APP_INSTALL: 'app.install',
+    ACTION_APP_RESTORE: 'app.restore',
+    ACTION_APP_UNINSTALL: 'app.uninstall',
+    ACTION_APP_UPDATE: 'app.update',
+    ACTION_BACKUP_FINISH: 'backup.finish',
+    ACTION_BACKUP_START: 'backup.start',
+    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CLI_MODE: 'settings.climode',
+    ACTION_START: 'cloudron.start',
+    ACTION_UPDATE: 'cloudron.update',
+    ACTION_USER_ADD: 'user.add',
+    ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_REMOVE: 'user.remove',
+    ACTION_USER_UPDATE: 'user.update'
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:eventlog'),
+    eventlogdb = require('./eventlogdb.js'),
+    util = require('util'),
+    uuid = require('node-uuid');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function EventLogError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(EventLogError, Error);
+EventLogError.INTERNAL_ERROR = 'Internal error';
+EventLogError.NOT_FOUND = 'Not Found';
+
+function add(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    var id = uuid.v4();
+
+    eventlogdb.add(id, action, source, data, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, { id: id });
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new EventLogError(EventLogError.NOT_FOUND, 'No such event'));
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.getAllPaged(action, search, page, perPage, function (error, boxes) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, boxes);
+    });
+}
@@ -0,0 +1,104 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getAllPaged: getAllPaged,
+    add: add,
+    count: count,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    mysql = require('mysql'),
+    safe = require('safetydance');
+
+var EVENTLOGS_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+
+// until mysql module supports automatic type coercion
+function postProcess(eventLog) {
+    eventLog.source = safe.JSON.parse(eventLog.source);
+    eventLog.data = safe.JSON.parse(eventLog.data);
+    return eventLog;
+}
+
+function get(eventId, callback) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE id = ?', [ eventId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, postProcess(result[0]));
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [];
+    var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog';
+
+    if (action || search) query += ' WHERE';
+    if (search) query += ' data LIKE ' + mysql.escape('%' + search + '%');
+    if (action && search) query += ' AND ';
+
+    if (action) {
+        query += ' action=?';
+        data.push(action);
+    }
+
+    query += ' ORDER BY creationTime DESC LIMIT ?,?';
+
+    data.push((page-1)*perPage);
+    data.push(perPage);
+
+    database.query(query, data, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, action, source, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM eventlog', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM eventlog', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
+
@@ -4,6 +4,7 @@ exports = module.exports = {
    get: get,
    getWithMembers: getWithMembers,
    getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
    add: add,
    del: del,
    count: count,
@@ -65,6 +66,19 @@ function getAll(callback) {
    });
 }

+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' GROUP BY groups.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
 function add(id, name, callback) {
    assert.strictEqual(typeof id, 'string');
    assert.strictEqual(typeof name, 'string');
@@ -10,6 +10,7 @@ exports = module.exports = {
    get: get,
    getWithMembers: getWithMembers,
    getAll: getAll,
+    getAllWithMembers: getAllWithMembers,

    getMembers: getMembers,
    addMember: addMember,
@@ -51,7 +52,7 @@ util.inherits(GroupError, Error);
 GroupError.INTERNAL_ERROR = 'Internal Error';
 GroupError.ALREADY_EXISTS = 'Already Exists';
 GroupError.NOT_FOUND = 'Not Found';
-GroupError.BAD_NAME = 'Bad name';
+GroupError.BAD_FIELD = 'Field error';
 GroupError.NOT_EMPTY = 'Not Empty';
 GroupError.NOT_ALLOWED = 'Not Allowed';

@@ -59,12 +60,12 @@ function validateGroupname(name) {
    assert.strictEqual(typeof name, 'string');
    var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group

-    if (name.length <= 2) return new GroupError(GroupError.BAD_NAME, 'name must be atleast 2 chars');
-    if (name.length >= 200) return new GroupError(GroupError.BAD_NAME, 'name too long');
+    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');

-    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_NAME, 'name can only have A-Za-z0-9_-');
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');

-    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_NAME, 'name is reserved');
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');

    return null;
 }
@@ -133,6 +134,16 @@ function getAll(callback) {
    });
 }

+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
 function getMembers(groupId, callback) {
    assert.strictEqual(typeof groupId, 'string');
    assert.strictEqual(typeof callback, 'function');
@@ -0,0 +1,21 @@
+'use strict';
+
+// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+// These constants are used in the installer script as well
+// Do not require anything here!
+
+exports = module.exports = {
+    'version': 34,
+
+    'baseImage': 'cloudron/base:0.8.1',
+
+    'images': {
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.11.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.10.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.9.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.8.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.13.1' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.8.0' }
+    }
+};
+
@@ -9,9 +9,12 @@ var assert = require('assert'),
    apps = require('./apps.js'),
    config = require('./config.js'),
    debug = require('debug')('box:ldap'),
+    eventlog = require('./eventlog.js'),
    user = require('./user.js'),
    UserError = user.UserError,
-    ldap = require('ldapjs');
+    ldap = require('ldapjs'),
+    mailboxes = require('./mailboxes.js'),
+    MailboxError = mailboxes.MailboxError;

 var gServer = null;

@@ -39,120 +42,227 @@ function getAppByRequest(req, callback) {
    });
 }

+function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // send user objects
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+            var groups = [ GROUP_USERS_DN ];
+            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+            var displayName = entry.displayName || entry.username;
+            var nameParts = displayName.split(' ');
+            var firstName = nameParts[0];
+            var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['user'],
+                    objectcategory: 'person',
+                    cn: entry.id,
+                    uid: entry.id,
+                    mail: entry.email,
+                    // TODO: check mailboxes before we send this
+                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    displayname: displayName,
+                    givenName: firstName,
+                    username: entry.username,
+                    samaccountname: entry.username,      // to support ActiveDirectory clients
+                    memberof: groups
+                }
+            };
+
+            // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+            // which is required to have atleast one character if present
+            if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result){
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        var groups = [{
+            name: 'users',
+            admin: false
+        }, {
+            name: 'admins',
+            admin: true
+        }];
+
+        groups.forEach(function (group) {
+            var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: group.name,
+                    memberuid: members.map(function(entry) { return entry.id; })
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function mailboxSearch(req, res, next) {
+    debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
+
+            // TODO: send aliases
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['mailbox'],
+                    objectcategory: 'mailbox',
+                    cn: entry.name,
+                    uid: entry.name,
+                    mail: entry.name + '@' + config.fqdn()
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function authenticateUser(req, res, next) {
+    debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+    // extract the common name which might have different attribute names
+    var attributeName = Object.keys(req.dn.rdns[0])[0];
+    var commonName = req.dn.rdns[0][attributeName];
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    var api;
+    if (attributeName === 'mail') {
+        api = user.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        var parts = commonName.split('@');
+        if (parts[1] === config.fqdn()) { // internal email, verify with username
+            commonName = parts[0];
+            api = user.verify;
+        } else { // external email
+            api = user.verifyWithEmail;
+        }
+    } else if (commonName.indexOf('uid-') === 0) {
+        api = user.verify;
+    } else {
+        api = user.verifyWithUsername;
+    }
+
+    api(commonName, req.credentials || '', function (error, user) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorizeUserForApp(req, res, next) {
+    assert(req.user);
+
+    // We simply authorize the user to access a mailbox by his own name
+    getAppByRequest(req, function (error, app) {
+        if (error) return next(error);
+
+        if (!app) {
+            debug('no app found for this container, allow access');
+            return res.end();
+        }
+
+        apps.hasAccessTo(app, req.user, function (error, result) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we return no such object, to avoid leakage of a users existence
+            if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: req.user.id });
+
+            res.end();
+        });
+    });
+}
+
+function authorizeUserForMailbox(req, res, next) {
+    assert(req.user);
+
+    mailboxes.get(req.user.username, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+
+        res.end();
+    });
+}
+
 function start(callback) {
    assert.strictEqual(typeof callback, 'function');

    gServer = ldap.createServer({ log: gLogger });

-    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+    gServer.search('ou=users,dc=cloudron', userSearch);
+    gServer.search('ou=groups,dc=cloudron', groupSearch);
+    gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);

-        user.list(function (error, result) {
-            if (error) return next(new ldap.OperationsError(error.toString()));
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);

-            // send user objects
-            result.forEach(function (entry) {
-                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
-
-                var groups = [ GROUP_USERS_DN ];
-                if (entry.admin) groups.push(GROUP_ADMINS_DN);
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['user'],
-                        objectcategory: 'person',
-                        cn: entry.id,
-                        uid: entry.id,
-                        mail: entry.email,
-                        displayname: entry.displayName || entry.username,
-                        username: entry.username,
-                        samaccountname: entry.username,      // to support ActiveDirectory clients
-                        memberof: groups
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
-
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
-
-            var groups = [{
-                name: 'users',
-                admin: false
-            }, {
-                name: 'admins',
-                admin: true
-            }];
-
-            groups.forEach(function (group) {
-                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['group'],
-                        cn: group.name,
-                        memberuid: members.map(function(entry) { return entry.id; })
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
-        // TODO: validate password
-        debug('ldap application bind: %s', req.dn.toString());
+    // this is the bind for addons (after bind, they might search and authenticate)
+    gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
+        debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
        res.end();
    });

-    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('ldap user bind: %s', req.dn.toString());
-
-        // extract the common name which might have different attribute names
-        var commonName = req.dn.rdns[0][Object.keys(req.dn.rdns[0])[0]];
-        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-        // TODO this should be done after we verified the app has access to avoid leakage of user existence
-        user.verify(commonName, req.credentials || '', function (error, userObject) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-            if (error) return next(new ldap.OperationsError(error));
-
-            getAppByRequest(req, function (error, app) {
-                if (error) return next(error);
-
-                if (!app) return res.end();
-
-                debug('no app found for this container, allow access');
-
-                apps.hasAccessTo(app, userObject, function (error, result) {
-                    if (error) return next(new ldap.OperationsError(error.toString()));
-
-                    // we return no such object, to avoid leakage of a users existence
-                    if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-                    res.end();
-                });
-            });
-        });
+    // this is the bind for apps (after bind, they might search and authenticate user)
+    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
+        // TODO: validate password
+        debug('application bind: %s', req.dn.toString());
+        res.end();
    });

    gServer.listen(config.get('ldapPort'), callback);
@@ -0,0 +1,37 @@
+'use strict';
+
+exports = module.exports = {
+    sendFailureLogs: sendFailureLogs
+};
+
+var assert = require('assert'),
+    mailer = require('./mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
+
+function collectLogs(unitName, callback) {
+    assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendFailureLogs(processName, options) {
+    assert.strictEqual(typeof processName, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    collectLogs(options.unit || processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending failure logs for', processName);
+
+        mailer.unexpectedExit(processName, result);
+    });
+}
@@ -0,0 +1,16 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+<% if (message) { %>
+<%= domain %> was not renewed.
+
+<%- message %>
+<% } else { %>
+<%= domain %> was renewed.
+<% } %>
+Thank you,
+Your Cloudron
+<% } else { %>
+
+<% } %>
+
@@ -1,6 +1,6 @@
 <%if (format === 'text') { %>

-Dear <%= username %>,
+Dear <%= user.username || user.email %>,

 Someone, hopefully you, has requested your <%= fqdn %>'s account password
 be reset. If you did not request this reset, please ignore this message.
@@ -2,7 +2,7 @@

 Dear Cloudron Team,

-Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> exited unexpectedly!

 Please see some excerpt of the logs below.

@@ -2,16 +2,16 @@

 Dear Admin,

-User with name '<%= username %>' (<%= email %>) was added in the Cloudron at <%= fqdn %>.
+User with email <%= user.email %> was added in the Cloudron at <%= fqdn %>.

 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.

 <% if (inviteLink) { %>
-This user was not invited immediately, he has to get invited manually later, using the "send invite" button in the admin panel.
-To perform any configuration on behalf of the user, please use this link
+As requested, this user has not been sent an invitation email.
+
+To set a password and perform any configuration on behalf of the user, please use this link:
 <%= inviteLink %>
-It allows to setup a temporary password, which the user will be able to override, once he gets invited.
-This link will become invalid as soon as the user was invited.
+
 <% } %>

 Thank you,
@@ -2,7 +2,7 @@

 Dear Admin,

-User with name '<%= username %>' (<%= email %>) <%= event %> in the Cloudron at <%= fqdn %>.
+User <%= user.username %> <%= user.email %> <%= event %> in the Cloudron at <%= fqdn %>.

 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.

@@ -1,20 +1,15 @@
 <%if (format === 'text') { %>

-Dear <%= user.username %>,
+Dear <%= user.email %>,

 Welcome to our Cloudron <%= fqdn %>!

 The Cloudron is our own Smart Server. You can read more about it
 at https://www.cloudron.io.

-           You username is '<%= user.username %>'
-
 To get started, create your account by visiting the following page:
 <%= setupLink %>

-When you visit the above page, you will be prompted to enter a new password.
-After you have submitted the form, you can login using the new password.
-
 <% if (invitor && invitor.email) { %>
 Thank you,
 <%= invitor.email %>
@@ -0,0 +1,127 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    getAliases: getAliases,
+    setAliases: setAliases,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'aliasTarget', 'creationTime' ].join(',');
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name) VALUES (?)', [ name ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE name=? OR aliasTarget = ?', [ name, name ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function postProcess(result) {
+    result.aliases = result.aliases ? result.aliases.split(',') : [ ];
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.name=? AND m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(results[0]);
+
+        callback(null, results[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ?', args: [ name ] });
+    aliases.forEach(function (alias) {
+        queries.push({ query: 'INSERT INTO mailboxes (name, aliasTarget) VALUES (?, ?)', args: [ alias, name ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name FROM mailboxes WHERE aliasTarget=? ORDER BY name', [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results = results.map(function (r) { return r.name; });
+        callback(null, results);
+    });
+}
@@ -0,0 +1,198 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    setAliases: setAliases,
+    getAliases: getAliases,
+
+    setupAliases: setupAliases,
+
+    MailboxError: MailboxError
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:mailboxes'),
+    docker = require('./docker.js'),
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util');
+
+function MailboxError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(MailboxError, Error);
+MailboxError.ALREADY_EXISTS = 'already exists';
+MailboxError.BAD_FIELD = 'Field error';
+MailboxError.NOT_FOUND = 'not found';
+MailboxError.INTERNAL_ERROR = 'internal error';
+MailboxError.EXTERNAL_ERROR = 'external error';
+
+function validateName(name) {
+    var RESERVED_NAMES = [ 'no-reply', 'postmaster', 'mailer-daemon' ];
+
+    if (!name.length) return new MailboxError(MailboxError.BAD_FIELD, "name cannot be empty");
+
+    if (name.length < 2) return new MailboxError(MailboxError.BAD_FIELD, 'name too small');
+    if (name.length > 127) return new MailboxError(MailboxError.BAD_FIELD, 'name too long');
+    if (RESERVED_NAMES.indexOf(name) !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'name is reserved');
+
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailboxError(MailboxError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    if (name.indexOf('.app') !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'alias pattern is reserved for apps');
+
+    return null;
+}
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.add(name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        debug('Added mailbox %s', name);
+
+        var mailbox = {
+            name: name
+        };
+
+        callback(null, mailbox);
+    });
+}
+
+function pushAlias(name, aliases, callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    var cmd = [ '/addons/mail/service.sh', 'set-alias', name ].concat(aliases);
+
+    debug('pushing alias for %s : %j', name, aliases);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(new MailboxError(MailboxError.EXTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    pushAlias(name, [ ], function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.del(name, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+            debug('deleted mailbox %s', name);
+
+            callback();
+        });
+    });
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.get(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, mailbox);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAll(function (error, results) {
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var i = 0; i < aliases.length; i++) {
+        aliases[i] = aliases[i].toLowerCase();
+
+        var error = validateName(aliases[i]);
+        if (error) return callback(error);
+    }
+
+    pushAlias(name, aliases, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.setAliases(name, aliases, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS, error.message))
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+
+            callback(null);
+        });
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAliases(name, function (error, aliases) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, aliases);
+    });
+}
+
+// push aliases to the mail container on startup
+function setupAliases(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, mailboxes) {
+        if (error) return callback(error);
+
+        async.each(mailboxes, function (mailbox, iteratorDone) {
+            getAliases(mailbox.name, function (error, aliases) {
+                if (error) return iteratorDone(error);
+
+                if (aliases.length === 0) return iteratorDone();
+
+                pushAlias(mailbox.name, aliases, iteratorDone);
+            });
+        }, callback)
+    });
+}
+
@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';

 exports = module.exports = {
@@ -14,16 +12,19 @@ exports = module.exports = {
    appUpdateAvailable: appUpdateAvailable,

    sendInvite: sendInvite,
-    sendCrashNotification: sendCrashNotification,
+    unexpectedExit: unexpectedExit,

    appDied: appDied,

    outOfDiskSpace: outOfDiskSpace,

+    certificateRenewed: certificateRenewed,
+
    FEEDBACK_TYPE_FEEDBACK: 'feedback',
    FEEDBACK_TYPE_TICKET: 'ticket',
    FEEDBACK_TYPE_APP_MISSING: 'app_missing',
    FEEDBACK_TYPE_APP_ERROR: 'app_error',
+    FEEDBACK_TYPE_UPGRADE_REQUEST: 'upgrade_request',
    sendFeedback: sendFeedback,

    _getMailQueue: _getMailQueue,
@@ -40,6 +41,7 @@ var assert = require('assert'),
    ejs = require('ejs'),
    nodemailer = require('nodemailer'),
    path = require('path'),
+    platform = require('./platform.js'),
    safe = require('safetydance'),
    smtpTransport = require('nodemailer-smtp-transport'),
    users = require('./user.js'),
@@ -108,6 +110,7 @@ function getTxtRecords(callback) {
    });
 }

+// keep this in sync with the cloudron.js dns changes
 function checkDns() {
    getTxtRecords(function (error, records) {
        if (error || !records) {
@@ -121,7 +124,7 @@ function checkDns() {
        for (var i = 0; i < records.length; i++) {
            if (records[i].indexOf('v=spf1 ') !== 0) continue; // not SPF

-            allowedToSendMail = records[i].indexOf('a:' + config.fqdn()) !== -1;
+            allowedToSendMail = records[i].indexOf('a:' + config.adminFqdn()) !== -1;
            break; // only one SPF record can exist (https://support.google.com/a/answer/4568483?hl=en)
        }

@@ -157,10 +160,14 @@ function sendMails(queue) {

        var transport = nodemailer.createTransport(smtpTransport({
            host: mailServerIp,
-            port: 2500 // this value comes from mail container
+            port: config.get('smtpPort'),
+            auth: {
+                user: platform.mailConfig().username,
+                pass: platform.mailConfig().password
+            }
        }));

-        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
+        debug('Processing mail queue of size %d (through %s:2525)', queue.length, mailServerIp);

        async.mapSeries(queue, function iterator(mailOptions, callback) {
            transport.sendMail(mailOptions, function (error) {
@@ -197,6 +204,8 @@ function getAdminEmails(callback) {
    users.getAllAdmins(function (error, admins) {
        if (error) return callback(error);

+        if (admins.length === 0) return callback(new Error('No admins on this cloudron')); // box not activated yet
+
        var adminEmails = [ ];
        admins.forEach(function (admin) { adminEmails.push(admin.email); });

@@ -214,10 +223,10 @@ function mailUserEventToAdmins(user, event) {
        adminEmails = _.difference(adminEmails, [ user.email ]);

        var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
            to: adminEmails.join(', '),
-            subject: util.format('%s %s in Cloudron %s', user.username, event, config.fqdn()),
-            text: render('user_event.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, event: event, format: 'text' }),
+            subject: util.format('%s %s in Cloudron %s', user.username || user.email, event, config.fqdn()),
+            text: render('user_event.ejs', { fqdn: config.fqdn(), user: user, event: event, format: 'text' }),
        };

        enqueue(mailOptions);
@@ -233,14 +242,14 @@ function sendInvite(user, invitor) {
    var templateData = {
        user: user,
        webadminUrl: config.adminOrigin(),
-        setupLink: config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken,
+        setupLink: config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken,
        format: 'text',
        fqdn: config.fqdn(),
        invitor: invitor
    };

    var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
        to: user.email,
        subject: util.format('Welcome to Cloudron %s', config.fqdn()),
        text: render('welcome_user.ejs', templateData)
@@ -260,25 +269,25 @@ function userAdded(user, inviteSent) {

        adminEmails = _.difference(adminEmails, [ user.email ]);

-        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken;
+        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken;

        var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
            to: adminEmails.join(', '),
-            subject: util.format('%s added in Cloudron %s', user.username, config.fqdn()),
-            text: render('user_added.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, inviteLink: inviteLink, format: 'text' }),
+            subject: util.format('%s added in Cloudron %s', user.email, config.fqdn()),
+            text: render('user_added.ejs', { fqdn: config.fqdn(), user: user, inviteLink: inviteLink, format: 'text' }),
        };

        enqueue(mailOptions);
    });
 }

-function userRemoved(username) {
-    assert.strictEqual(typeof username, 'string');
+function userRemoved(user) {
+    assert.strictEqual(typeof user, 'object');

-    debug('Sending mail for userRemoved');
+    debug('Sending mail for userRemoved.', user.id, user.email);

-    mailUserEventToAdmins({ username: username }, 'was removed');
+    mailUserEventToAdmins(user, 'was removed');
 }

 function adminChanged(user, admin) {
@@ -293,15 +302,15 @@ function adminChanged(user, admin) {
 function passwordReset(user) {
    assert.strictEqual(typeof user, 'object');

-    debug('Sending mail for password reset for user %s.', user.username);
+    debug('Sending mail for password reset for user %s.', user.email, user.id);

    var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;

    var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
        to: user.email,
        subject: 'Password Reset Request',
-        text: render('password_reset.ejs', { fqdn: config.fqdn(), username: user.username, resetLink: resetLink, format: 'text' })
+        text: render('password_reset.ejs', { fqdn: config.fqdn(), user: user, resetLink: resetLink, format: 'text' })
    };

    enqueue(mailOptions);
@@ -316,7 +325,7 @@ function appDied(app) {
        if (error) return console.log('Error getting admins', error);

        var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
            to: adminEmails.concat('support@cloudron.io').join(', '),
            subject: util.format('App %s is down', app.location),
            text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
@@ -334,7 +343,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
        if (error) return console.log('Error getting admins', error);

         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
            to: adminEmails.join(', '),
            subject: util.format('%s has a new update available', config.fqdn()),
            text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
@@ -352,7 +361,7 @@ function appUpdateAvailable(app, updateInfo) {
        if (error) return console.log('Error getting admins', error);

         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
            to: adminEmails.join(', '),
            subject: util.format('%s has a new update available', app.fqdn),
            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
@@ -366,7 +375,7 @@ function outOfDiskSpace(message) {
    assert.strictEqual(typeof message, 'string');

    var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
        to: 'admin@cloudron.io',
        subject: util.format('[%s] Out of disk space alert', config.fqdn()),
        text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
@@ -375,17 +384,31 @@ function outOfDiskSpace(message) {
    sendMails([ mailOptions ]);
 }

+function certificateRenewed(domain, message) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof message, 'string');
+
+    var mailOptions = {
+        from: platform.mailConfig().from,
+        to: 'admin@cloudron.io',
+        subject: util.format('[%s] Certificate was %s renewed', domain, message ? 'not' : ''),
+        text: render('certificate_renewed.ejs', { domain: domain, message: message, format: 'text' })
+    };
+
+    sendMails([ mailOptions ]);
+}
+
 // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
 // crashnotifier should be able to send mail when there is no db
-function sendCrashNotification(program, context) {
+function unexpectedExit(program, context) {
    assert.strictEqual(typeof program, 'string');
    assert.strictEqual(typeof context, 'string');

    var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
        to: 'admin@cloudron.io',
        subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
-        text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
+        text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
    };

    sendMails([ mailOptions ]);
@@ -400,10 +423,11 @@ function sendFeedback(user, type, subject, description) {
    assert(type === exports.FEEDBACK_TYPE_TICKET ||
        type === exports.FEEDBACK_TYPE_FEEDBACK ||
        type === exports.FEEDBACK_TYPE_APP_MISSING ||
+        type === exports.FEEDBACK_TYPE_UPGRADE_REQUEST ||
        type === exports.FEEDBACK_TYPE_APP_ERROR);

    var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
        to: 'support@cloudron.io',
        subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
        text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})
@@ -4,7 +4,6 @@ exports = module.exports = {
    cookieParser: require('cookie-parser'),
    cors: require('./cors'),
    csrf: require('csurf'),
-    favicon: require('serve-favicon'),
    json: require('body-parser').json,
    morgan: require('morgan'),
    proxy: require('proxy-middleware'),
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 var assert = require('assert'),
@@ -13,6 +11,7 @@ var assert = require('assert'),
    shell = require('./shell.js');

 exports = module.exports = {
+    requiresOAuthProxy: requiresOAuthProxy,
    configureAdmin: configureAdmin,
    configureApp: configureApp,
    unconfigureApp: unconfigureApp,
@@ -22,6 +21,19 @@ exports = module.exports = {
 var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');

+function requiresOAuthProxy(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    var tmp = app.accessRestriction;
+
+    // if no accessRestriction set, or the app uses one of the auth modules, we do not need the oauth proxy
+    if (tmp === null) return false;
+    if (app.manifest.addons['ldap'] || app.manifest.addons['oauth'] || app.manifest.addons['simpleauth']) return false;
+
+    // check if any restrictions are set
+    return !!((tmp.users && tmp.users.length) || (tmp.groups && tmp.groups.length));
+}
+
 function configureAdmin(certFilePath, keyFilePath, callback) {
    assert.strictEqual(typeof certFilePath, 'string');
    assert.strictEqual(typeof keyFilePath, 'string');
@@ -43,16 +55,16 @@ function configureAdmin(certFilePath, keyFilePath, callback) {
    reload(callback);
 }

-function configureApp(app, oauthProxy, certFilePath, keyFilePath, callback) {
+function configureApp(app, certFilePath, keyFilePath, callback) {
    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
    assert.strictEqual(typeof certFilePath, 'string');
    assert.strictEqual(typeof keyFilePath, 'string');
    assert.strictEqual(typeof callback, 'function');

    var sourceDir = path.resolve(__dirname, '..');
+    var oauthProxy = requiresOAuthProxy(app);
    var endpoint = oauthProxy ? 'oauthproxy' : 'app';
-    var vhost = config.appFqdn(app.location);
+    var vhost = app.altDomain || config.appFqdn(app.location);

    var data = {
        sourceDir: sourceDir,
@@ -66,10 +78,10 @@ function configureApp(app, oauthProxy, certFilePath, keyFilePath, callback) {
    var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);

    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writing config for "%s" to %s', app.location, nginxConfigFilename);
+    debug('writing config for "%s" to %s', vhost, nginxConfigFilename);

    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-        debug('Error creating nginx config for "%s" : %s', app.location, safe.error.message);
+        debug('Error creating nginx config for "%s" : %s', vhost, safe.error.message);
        return callback(safe.error);
    }

@@ -80,9 +92,11 @@ function unconfigureApp(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

+    var vhost = app.altDomain || config.appFqdn(app.location);
+
    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
    if (!safe.fs.unlinkSync(nginxConfigFilename)) {
-        debug('Error removing nginx configuration of "%s": %s', app.location, safe.error.message);
+        debug('Error removing nginx configuration of "%s": %s', vhost, safe.error.message);
        return callback(null);
    }

@@ -0,0 +1,78 @@
+<% include header %>
+
+<!-- tester -->
+
+<script>
+
+'use strict';
+
+// very basic angular app
+var app = angular.module('Application', []);
+app.controller('Controller', ['$scope', function ($scope) {
+    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
+    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
+}]);
+
+</script>
+
+<center>
+    <br/>
+    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to your Cloudron.</h4>
+    <h2>Setup your account and password.</h2>
+</center>
+
+<div class="container" ng-app="Application" ng-controller="Controller">
+    <div class="row">
+        <div class="col-md-6 col-md-offset-3">
+            <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
+                <input type="password" style="display: none;">
+                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
+
+                <center><p class="has-error"><%= error %></p></center>
+
+<% if (user && user.username) { %>
+                <div class="form-group"">
+                    <label class="control-label">Username</label>
+                    <input type="text" class="form-control" ng-model="username" name="username" readonly required>
+                </div>
+<% } else { %>
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
+                    <label class="control-label">Username</label>
+                    <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
+                        <small ng-show="setupForm.username.$error.minlength">The username is too short</small>
+                        <small ng-show="setupForm.username.$error.maxlength">The username is too long</small>
+                        <small ng-show="setupForm.username.$dirty && setupForm.username.$invalid">Not a valid username</small>
+                    </div>
+                    <input type="text" class="form-control" ng-model="username" name="username" ng-maxlength="512" ng-minlength="3" required autofocus>
+                </div>
+<% } %>
+
+                <div class="form-group">
+                    <label class="control-label">Display Name</label>
+                    <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
+                </div>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
+                    <label class="control-label">New Password</label>
+                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
+                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+                    </div>
+                    <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" required>
+                </div>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
+                    <label class="control-label">Repeat Password</label>
+                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
+                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
+                    </div>
+                    <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
+                </div>
+
+                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
+            </form>
+        </div>
+    </div>
+</div>
+
+<% include footer %>
@@ -8,9 +8,11 @@

    <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">

+    <!-- Theme CSS -->
+    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
+
    <!-- Custom Fonts -->
    <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
-    <link href="https://fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">

    <!-- jQuery-->
    <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
@@ -22,9 +24,6 @@
    <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
    <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>

-    <!-- Theme CSS -->
-    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
-
 </head>

 <body class="oauth">
@@ -13,13 +13,14 @@ app.controller('Controller', [function () {}]);
 </script>

 <center>
-    <h1>Hello <%= user.username %> create a new password</h1>
+    <h1>Hello <%= user.username %>, set a new password</h1>
 </center>

 <div class="container" ng-app="Application" ng-controller="Controller">
    <div class="row">
        <div class="col-md-6 col-md-offset-3">
            <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
+                <input type="password" style="display: none;">
                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>

@@ -3,14 +3,14 @@
 <!-- tester -->

 <center>
-    <h1>Reset your password successful</h1>
+    <h1>Password reset successful</h1>
 </center>

 <div class="container">
    <div class="row">
        <div class="col-md-6 col-md-offset-3">
-            <p>An email was sent to you with a link to create a new password.</p>
-            If you have not received any email after some time, maybe you have misspelled your email address, simply try again <a href="/api/v1/session/password/resetRequest.html">here</a>.
+            <p>An email was sent to you with a link to set a new password.</p>
+            If you have not received any email, simply <a href="/api/v1/session/password/resetRequest.html">try again</a>.
        </div>
    </div>
 </div>
@@ -1,46 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
-
-</script>
-
-<center>
-    <h1>Hello <%= user.username %>, set a password</h1>
-</center>
-
-<div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-        <div class="col-md-6 col-md-offset-3">
-            <form action="/api/v1/session/password/reset" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
-                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-                <div class="form-group" ng-class="{ 'has-error': setupForm.password.$dirty && setupForm.password.$invalid }">
-                    <label class="control-label" for="inputPassword">New Password</label>
-                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
-                    </div>
-                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" autofocus required>
-                </div>
-                <div class="form-group" ng-class="{ 'has-error': setupForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
-                    <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
-                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-                    </div>
-                    <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
-                </div>
-                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
-            </form>
-        </div>
-    </div>
-</div>
-
-<% include footer %>
@@ -7,7 +7,7 @@ exports = module.exports = {

 var appdb = require('./appdb.js'),
    assert = require('assert'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
    config = require('./config.js'),
    DatabaseError = require('./databaseerror.js'),
    debug = require('debug')('box:proxy'),
@@ -124,7 +124,7 @@ function authenticate(req, res, next) {
                return res.send(500, 'Unknown app.');
            }

-            clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
+            clients.getByAppIdAndType(result.id, clients.TYPE_PROXY, function (error, result) {
                if (error) {
                    console.error('Unknown OAuth client.', error);
                    return res.send(500, 'Unknown OAuth client.');
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 var config = require('./config.js'),
@@ -13,8 +11,6 @@ exports = module.exports = {

    ADDON_CONFIG_DIR: path.join(config.baseDir(), 'data/addons'),

-    DNS_IN_SYNC_FILE: path.join(config.baseDir(), 'data/dns_in_sync'),
-
    COLLECTD_APPCONFIG_DIR: path.join(config.baseDir(), 'data/collectd/collectd.conf.d'),

    DATA_DIR: path.join(config.baseDir(), 'data'),
@@ -26,9 +22,12 @@ exports = module.exports = {

    CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'data/box/avatar.png'),
    CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
+    FIRST_RUN_FILE: path.join(config.baseDir(), 'data/box/first_run'),

    UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),

    ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
-    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key')
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key'),
+
+    INFRA_VERSION_FILE: path.join(config.baseDir(), 'data/INFRA_VERSION')
 };
@@ -0,0 +1,113 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+
+    mailConfig: mailConfig
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    certificates = require('./certificates.js'),
+    debug = require('debug')('box:platform'),
+    fs = require('fs'),
+    infra = require('./infra_version.js'),
+    ini = require('ini'),
+    mailboxes = require('./mailboxes.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    util = require('util');
+
+var SETUP_INFRA_CMD = path.join(__dirname, 'scripts/setup_infra.sh');
+
+var gAddonVars = null;
+
+function initialize(callback) {
+    if (process.env.BOX_ENV === 'test' && !process.env.CREATE_INFRA) return callback();
+
+    debug('initializing addon infrastructure');
+
+    var existingInfra = { version: 'none' };
+    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
+        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
+        if (!existingInfra) existingInfra = { version: 'legacy' };
+    }
+
+    if (infra.version === existingInfra.version) {
+        debug('platform is uptodate at version %s', infra.version);
+        return loadAddonVars(callback);
+    }
+
+    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
+
+    async.series([
+        stopContainers,
+        startAddons,
+        removeOldImages,
+        existingInfra.version === 'none' ? apps.restoreInstalledApps : apps.configureInstalledApps,
+        loadAddonVars,
+        mailboxes.setupAliases,
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra))
+    ], callback);
+}
+
+function removeOldImages(callback) {
+    debug('removing old addon images');
+
+    for (var imageName in infra.images) {
+        var image = infra.images[imageName];
+        debug('cleaning up images of %s', image);
+        var cmd = 'docker images "%s" | tail -n +2 | awk \'{ print $1 ":" $2 }\' | grep -v "%s" | xargs --no-run-if-empty docker rmi';
+        shell.execSync('removeOldImagesSync', util.format(cmd, image.repo, image.tag));
+    }
+
+    callback();
+}
+
+function removeImagesSync() {
+    debug('removing existing images');
+    shell.execSync('removeImagesSync', 'docker images -q | xargs --no-run-if-empty docker rmi -f');
+}
+
+function stopContainers(callback) {
+    // TODO: be nice and stop addons cleanly (example, shutdown commands)
+    debug('stopping existing containers');
+    shell.execSync('stopContainersSync', 'docker ps -qa | xargs --no-run-if-empty docker rm -f');
+    callback();
+}
+
+function startAddons(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
+        if (error) return callback(error);
+
+        shell.sudo('seutp_infra', [ SETUP_INFRA_CMD, paths.DATA_DIR, config.fqdn(), config.adminFqdn(), certFilePath, keyFilePath ], function (error) {
+            callback(error);
+        });
+    });
+}
+
+function loadAddonVars(callback) {
+    gAddonVars = {
+        mail: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mail_vars.sh', 'utf8')),
+        postgresql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'utf8')),
+        mysql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 'utf8')),
+        mongodb: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'utf8'))
+    };
+    callback();
+}
+
+function mailConfig() {
+    if (!gAddonVars) return { username: 'no-reply', from: 'no-reply@' + config.fqdn(), password: 'doesnotwork' }; // for tests which don't run infra
+
+    return {
+        username: gAddonVars.mail.MAIL_ROOT_USERNAME,
+        from: '"Cloudron" <' + gAddonVars.mail.MAIL_ROOT_USERNAME + '@' + config.fqdn() + '>',
+        password: gAddonVars.mail.MAIL_ROOT_PASSWORD
+    };
+}
@@ -1,10 +1,7 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
    getApp: getApp,
-    getAppBySubdomain: getAppBySubdomain,
    getApps: getApps,
    getAppIcon: getAppIcon,
    installApp: installApp,
@@ -31,8 +28,12 @@ var apps = require('../apps.js'),
    HttpSuccess = require('connect-lastmile').HttpSuccess,
    paths = require('../paths.js'),
    safe = require('safetydance'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}

 function removeInternalAppFields(app) {
    return {
@@ -49,7 +50,8 @@ function removeInternalAppFields(app) {
        portBindings: app.portBindings,
        iconUrl: app.iconUrl,
        fqdn: app.fqdn,
-        memoryLimit: app.memoryLimit
+        memoryLimit: app.memoryLimit,
+        altDomain: app.altDomain
    };
 }

@@ -64,17 +66,6 @@ function getApp(req, res, next) {
    });
 }

-function getAppBySubdomain(req, res, next) {
-    assert.strictEqual(typeof req.params.subdomain, 'string');
-
-    apps.getBySubdomain(req.params.subdomain, function (error, app) {
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such subdomain'));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(200, removeInternalAppFields(app)));
-    });
-}
-
 function getApps(req, res, next) {
    assert.strictEqual(typeof req.user, 'object');

@@ -98,80 +89,74 @@ function getAppIcon(req, res, next) {
    });
 }

-/*
- * Installs an app
- * @bodyparam {string} appStoreId The id of the app to be installed
- * @bodyparam {manifest} manifest The app manifest
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from environment variable name to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- * @bodyparam {icon} icon Base64 encoded image
- */
 function installApp(req, res, next) {
    assert.strictEqual(typeof req.body, 'object');

    var data = req.body;

-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId is required'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    // required
    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
+
+    // optional
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
    if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
    if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
    if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
    if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));

-    // allow tests to provide an appId for testing
-    var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
+    // falsy value in altDomain unsets it
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));

-    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j memoryLimit:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.memoryLimit, data.manifest);
+    debug('Installing app id:%s data:%j', data);

-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
+    apps.install(data, auditSource(req), function (error, app) {
        if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
        if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
        if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
        if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
        if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
        if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
        if (error) return next(new HttpError(500, error));

-        next(new HttpSuccess(202, { id: appId } ));
+        next(new HttpSuccess(202, app));
    });
 }

-/*
- * Configure an app
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from env to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- */
 function configureApp(req, res, next) {
    assert.strictEqual(typeof req.body, 'object');
    assert.strictEqual(typeof req.params.id, 'string');

    var data = req.body;

-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if ('location' in data && typeof data.location !== 'string') return next(new HttpError(400, 'location must be string'));
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    if ('accessRestriction' in data && typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
    if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
    if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
    if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));

-    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j memoryLimit:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.memoryLimit);
+    debug('Configuring app id:%s data:%j', req.params.id, data);

-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
+    apps.configure(req.params.id, data, auditSource(req), function (error) {
        if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
        if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
        if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -190,7 +175,7 @@ function restoreApp(req, res, next) {

    debug('Restore app id:%s', req.params.id);

-    apps.restore(req.params.id, function (error) {
+    apps.restore(req.params.id, auditSource(req), function (error) {
        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
        if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -215,16 +200,12 @@ function backupApp(req, res, next) {
    });
 }

-/*
- * Uninstalls an app
- * @bodyparam {string} id The id of the app to be uninstalled
- */
 function uninstallApp(req, res, next) {
    assert.strictEqual(typeof req.params.id, 'string');

    debug('Uninstalling app id:%s', req.params.id);

-    apps.uninstall(req.params.id, function (error) {
+    apps.uninstall(req.params.id, auditSource(req), function (error) {
        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
        if (error) return next(new HttpError(500, error));

@@ -266,15 +247,18 @@ function updateApp(req, res, next) {

    var data = req.body;

-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
    if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
    if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));

    debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);

-    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings, data.icon, function (error) {
+    apps.update(req.params.id, req.body, auditSource(req), function (error) {
        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
        if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -344,15 +328,37 @@ function getLogs(req, res, next) {
    });
 }

+function demuxStream(stream, stdin) {
+    var header = null;
+
+    stream.on('readable', function() {
+        header = header || stream.read(4);
+
+        while (header !== null) {
+            var length = header.readUInt32BE(0);
+            if (length === 0) {
+                header = null;
+                return stdin.end(); // EOF
+            }
+
+            var payload = stream.read(length);
+
+            if (payload === null) break;
+            stdin.write(payload);
+            header = stream.read(4);
+        }
+    });
+}
+
 function exec(req, res, next) {
    assert.strictEqual(typeof req.params.id, 'string');

-    debug('Execing into app id:%s', req.params.id);
+    debug('Execing into app id:%s and cmd:%s', req.params.id, req.query.cmd);

    var cmd = null;
    if (req.query.cmd) {
        cmd = safe.JSON.parse(req.query.cmd);
-        if (!util.isArray(cmd) && cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+        if (!util.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
    }

    var columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
@@ -373,15 +379,29 @@ function exec(req, res, next) {
        req.clearTimeout();
        res.sendUpgradeHandshake();

+        // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged.
        duplexStream.pipe(res.socket);
-        res.socket.pipe(duplexStream);
+
+        if (tty) {
+            res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit
+        } else {
+            demuxStream(res.socket, duplexStream);
+            res.socket.on('error', function () { duplexStream.end(); });
+            res.socket.on('end', function () { duplexStream.end(); });
+        }
    });
 }

 function listBackups(req, res, next) {
    assert.strictEqual(typeof req.params.id, 'string');

-    apps.listBackups(req.params.id, function (error, result) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    apps.listBackups(page, perPage, req.params.id, function (error, result) {
        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
        if (error) return next(new HttpError(500, error));

@@ -1,22 +1,30 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
    get: get,
-    create: create
+    create: create,
+    createDownloadUrl: createDownloadUrl
 };

-var backups = require('../backups.js'),
+var assert = require('assert'),
+    backups = require('../backups.js'),
    BackupsError = require('../backups.js').BackupsError,
-    cloudron = require('../cloudron.js'),
-    CloudronError = require('../cloudron.js').CloudronError,
-    debug = require('debug')('box:routes/backups'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess;

+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function get(req, res, next) {
-    backups.getAllPaged(1, 5, function (error, result) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    backups.getPaged(page, perPage, function (error, result) {
        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
        if (error) return next(new HttpError(500, error));

@@ -27,10 +35,20 @@ function get(req, res, next) {
 function create(req, res, next) {
    // note that cloudron.backup only waits for backup initiation and not for backup to complete
    // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    backups.backup(auditSource(req), function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
        if (error) return next(new HttpError(500, error));

        next(new HttpSuccess(202, {}));
    });
 }
+
+function createDownloadUrl(req, res, next) {
+    assert.strictEqual(typeof req.params.backupId, 'string');
+
+    backups.getRestoreUrl(req.params.backupId, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -12,7 +10,6 @@ exports = module.exports = {
 };

 var assert = require('assert'),
-    clientdb = require('../clientdb.js'),
    clients = require('../clients.js'),
    ClientsError = clients.ClientsError,
    DatabaseError = require('../databaseerror.js'),
@@ -29,8 +26,8 @@ function add(req, res, next) {
    if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
    if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));

-    clients.add(data.appId, clientdb.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, 'Invalid scope'));
+    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
+        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, error.message));
        if (error) return next(new HttpError(500, error));
        next(new HttpSuccess(201, result));
    });
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -10,19 +8,27 @@ exports = module.exports = {
    getProgress: getProgress,
    getConfig: getConfig,
    update: update,
-    feedback: feedback
+    feedback: feedback,
+    checkForUpdates: checkForUpdates
 };

 var assert = require('assert'),
+    async = require('async'),
    cloudron = require('../cloudron.js'),
-    config = require('../config.js'),
-    progress = require('../progress.js'),
-    mailer = require('../mailer.js'),
    CloudronError = cloudron.CloudronError,
+    config = require('../config.js'),
    debug = require('debug')('box:routes/cloudron'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    superagent = require('superagent');
+    progress = require('../progress.js'),
+    mailer = require('../mailer.js'),
+    superagent = require('superagent'),
+    updateChecker = require('../updatechecker.js');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}

 /**
 * Creating an admin user and activate the cloudron.
@@ -50,11 +56,9 @@ function activate(req, res, next) {
    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
    debug('activate: username:%s ip:%s', username, ip);

-    cloudron.activate(username, password, email, displayName, ip, function (error, info) {
+    cloudron.activate(username, password, email, displayName, ip, auditSource(req), function (error, info) {
        if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
-        if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
-        if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
-        if (error && error.reason === CloudronError.BAD_EMAIL) return next(new HttpError(400, 'Bad email'));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error) return next(new HttpError(500, error));

        // only in caas case do we have to notify the api server about activation
@@ -119,7 +123,7 @@ function getConfig(req, res, next) {

 function update(req, res, next) {
    // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    cloudron.updateToLatest(auditSource(req), function (error) {
        if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
        if (error) return next(new HttpError(500, error));
@@ -128,13 +132,23 @@ function update(req, res, next) {
    });
 }

+function checkForUpdates(req, res, next) {
+    async.series([
+        updateChecker.checkAppUpdates,
+        updateChecker.checkBoxUpdates
+    ], function () {
+        next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
+    });
+}
+
 function feedback(req, res, next) {
    assert.strictEqual(typeof req.user, 'object');

    if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK &&
        req.body.type !== mailer.FEEDBACK_TYPE_TICKET &&
        req.body.type !== mailer.FEEDBACK_TYPE_APP_MISSING &&
-        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app_missing" or "app_error"'));
+        req.body.type !== mailer.FEEDBACK_TYPE_UPGRADE_REQUEST &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback", "app_missing", "app_error" or "upgrade_request"'));
    if (typeof req.body.subject !== 'string' || !req.body.subject) return next(new HttpError(400, 'subject must be string'));
    if (typeof req.body.description !== 'string' || !req.body.description) return next(new HttpError(400, 'description must be string'));

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -15,8 +13,13 @@ var developer = require('../developer.js'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess;

+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function enabled(req, res, next) {
-    developer.enabled(function (error, enabled) {
+    developer.isEnabled(function (error, enabled) {
        if (enabled) return next();
        next(new HttpError(412, 'Developer mode not enabled'));
    });
@@ -25,8 +28,9 @@ function enabled(req, res, next) {
 function setEnabled(req, res, next) {
    if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled must be boolean'));

-    developer.setEnabled(req.body.enabled, function (error) {
+    developer.setEnabled(req.body.enabled, auditSource(req), function (error) {
        if (error) return next(new HttpError(500, error));
+
        next(new HttpSuccess(200, {}));
    });
 }
@@ -40,7 +44,7 @@ function login(req, res, next) {
        if (error) return next(new HttpError(500, error));
        if (!user) return next(new HttpError(401, 'Invalid credentials'));

-        developer.issueDeveloperToken(user, function (error, result) {
+        developer.issueDeveloperToken(user, auditSource(req), function (error, result) {
            if (error) return next(new HttpError(500, error));

            next(new HttpSuccess(200, { token: result.token, expiresAt: result.expiresAt }));
@@ -53,4 +57,4 @@ function apps(req, res, next) {
        if (error) return next(new HttpError(500, error));
        next(new HttpSuccess(200, { apps: result }));
    });
-}
+}
@@ -0,0 +1,26 @@
+'use strict';
+
+exports = module.exports = {
+    get: get
+};
+
+var eventlog = require('../eventlog.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
+
+function get(req, res, next) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    if (req.query.action && typeof req.query.action !== 'string') return next(new HttpError(400, 'action must be a string'));
+    if (req.query.search && typeof req.query.search !== 'string') return next(new HttpError(400, 'search must be a string'));
+
+    eventlog.getAllPaged(req.query.action || null, req.query.search || null, page, perPage, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { eventlogs: result }));
+    });
+}
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -22,7 +20,7 @@ function create(req, res, next) {
    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));

    groups.create(req.body.name, function (error, group) {
-        if (error && error.reason === GroupError.BAD_NAME) return next(new HttpError(400, error.message));
+        if (error && error.reason === GroupError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error && error.reason === GroupError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
        if (error) return next(new HttpError(500, error));

@@ -47,7 +45,7 @@ function get(req, res, next) {
 }

 function list(req, res, next) {
-    groups.getAll(function (error, result) {
+    groups.getAllWithMembers(function (error, result) {
        if (error) return next(new HttpError(500, error));

        next(new HttpSuccess(200, { groups: result }));
@@ -6,10 +6,13 @@ exports = module.exports = {
    clients: require('./clients.js'),
    cloudron: require('./cloudron.js'),
    developer: require('./developer.js'),
+    eventlog: require('./eventlog.js'),
    graphs: require('./graphs.js'),
    groups: require('./groups.js'),
-    internal: require('./internal.js'),
+    mailboxes: require('./mailboxes.js'),
    oauth2: require('./oauth2.js'),
+    profile: require('./profile.js'),
+    sysadmin: require('./sysadmin.js'),
    settings: require('./settings.js'),
    user: require('./user.js')
 };
@@ -0,0 +1,91 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    remove: remove,
+    create: create,
+    setAliases: setAliases,
+    getAliases: getAliases
+};
+
+var assert = require('assert'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    mailboxes = require('../mailboxes.js'),
+    MailboxError = mailboxes.MailboxError,
+    util = require('util');
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    mailboxes.add(req.body.name, function (error, mailbox) {
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, mailbox));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.get(req.params.mailboxId, function (error, result) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { mailboxes: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.del(req.params.mailboxId, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'Mailbox not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    if (!util.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
+
+    for (var i = 0; i < req.body.aliases.length; i++) {
+        if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
+    }
+
+    mailboxes.setAliases(req.params.mailboxId, req.body.aliases, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'One or more alias already exist'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+function getAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.getAliases(req.params.mailboxId, function (error, aliases) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { aliases: aliases }));
+    });
+}
@@ -1,34 +1,37 @@
-/* jslint  node:true */
-
 'use strict';

-var assert = require('assert'),
+var appdb = require('../appdb'),
    apps = require('../apps'),
+    assert = require('assert'),
    authcodedb = require('../authcodedb'),
-    clientdb = require('../clientdb'),
+    clients = require('../clients'),
    config = require('../config.js'),
    constants = require('../constants.js'),
    DatabaseError = require('../databaseerror'),
    debug = require('debug')('box:routes/oauth2'),
+    eventlog = require('../eventlog.js'),
+    hat = require('hat'),
    HttpError = require('connect-lastmile').HttpError,
    middleware = require('../middleware/index.js'),
    oauth2orize = require('oauth2orize'),
    passport = require('passport'),
    querystring = require('querystring'),
-    util = require('util'),
    session = require('connect-ensure-login'),
-    settings = require('../settings.js'),
    tokendb = require('../tokendb'),
-    appdb = require('../appdb'),
    url = require('url'),
    user = require('../user.js'),
    UserError = user.UserError,
-    hat = require('hat');
+    util = require('util'),
+    _ = require('underscore');
+
+function auditSource(req, appId) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { authType: 'oauth', ip: ip, appId: appId };
+}

 // create OAuth 2.0 server
 var gServer = oauth2orize.createServer();

-
 // Register serialialization and deserialization functions.
 //
 // The client id is stored in the session and can thus be retrieved for each
@@ -39,7 +42,7 @@ gServer.serializeClient(function (client, callback) {
 });

 gServer.deserializeClient(function (id, callback) {
-    clientdb.get(id, callback);
+    clients.get(id, callback);
 });


@@ -58,7 +61,7 @@ gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client,
    var code = hat(256);
    var expiresAt = Date.now() + 60 * 60000; // 1 hour

-    authcodedb.add(code, client.id, user.username, expiresAt, function (error) {
+    authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
        if (error) return callback(error);

        debug('grant code: new auth code for client %s code %s', client.id, code);
@@ -74,7 +77,7 @@ gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client,
    var token = tokendb.generateToken();
    var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day

-    tokendb.add(token, tokendb.PREFIX_USER + user.id, client.id, expires, client.scope, function (error) {
+    tokendb.add(token, user.id, client.id, expires, client.scope, function (error) {
        if (error) return callback(error);

        debug('grant token: new access token for client %s token %s', client.id, token);
@@ -104,7 +107,7 @@ gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI,
            var token = tokendb.generateToken();
            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day

-            tokendb.add(token, tokendb.PREFIX_USER + authCode.userId, authCode.clientId, expires, client.scope, function (error) {
+            tokendb.add(token, authCode.userId, authCode.clientId, expires, client.scope, function (error) {
                if (error) return callback(error);

                debug('exchange: new access token for client %s token %s', client.id, token);
@@ -199,13 +202,13 @@ function loginForm(req, res) {
        });
    }

-    clientdb.get(u.query.client_id, function (error, result) {
+    clients.get(u.query.client_id, function (error, result) {
        if (error) return sendError(req, res, 'Unknown OAuth client');

        switch (result.type) {
-            case clientdb.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
+            case clients.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
+            case clients.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
+            case clients.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
            default: break;
        }

@@ -268,19 +271,60 @@ function passwordSentSite(req, res) {
    renderTemplate(res, 'password_reset_sent', { adminOrigin: config.adminOrigin(), title: 'Cloudron Password Reset' });
 }

-// -> GET /api/v1/session/password/setup.html
-function passwordSetupSite(req, res, next) {
-    if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
+function renderAccountSetupSite(res, req, userObject, error) {
+    renderTemplate(res, 'account_setup', {
+        adminOrigin: config.adminOrigin(),
+        user: userObject,
+        error: error,
+        csrf: req.csrfToken(),
+        resetToken: req.query.reset_token || req.body.resetToken,
+        title: 'Cloudron Password Setup'
+    });
+}

-    user.getByResetToken(req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid reset_token'));
+// -> GET /api/v1/session/account/setup.html
+function accountSetupSite(req, res) {
+    if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');

-        renderTemplate(res, 'password_setup', {
-            adminOrigin: config.adminOrigin(),
-            user: user,
-            csrf: req.csrfToken(),
-            resetToken: req.query.reset_token,
-            title: 'Cloudron Password Setup'
+    user.getByResetToken(req.query.reset_token, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Reset Token');
+
+        renderAccountSetupSite(res, req, userObject, '');
+    });
+}
+
+// -> POST /api/v1/session/account/setup
+function accountSetup(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
+    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
+    if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
+
+    debug('acountSetup: with token %s.', req.body.resetToken);
+
+    user.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Reset Token');
+
+        var data = _.pick(req.body, 'username', 'displayName');
+        user.update(userObject.id, data, auditSource(req), function (error) {
+            if (error && error.reason === UserError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
+            if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+            if (error && error.reason === UserError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
+            if (error) return next(new HttpError(500, error));
+
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
+            // setPassword clears the resetToken
+            user.setPassword(userObject.id, req.body.password, function (error, result) {
+                if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+
+                if (error) return next(new HttpError(500, error));
+
+                res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
+            });
        });
    });
 }
@@ -314,9 +358,11 @@ function passwordReset(req, res, next) {
    user.getByResetToken(req.body.resetToken, function (error, userObject) {
        if (error) return next(new HttpError(401, 'Invalid resetToken'));

+        if (!userObject.username) return next(new HttpError(401, 'No username set'));
+
        // setPassword clears the resetToken
        user.setPassword(userObject.id, req.body.password, function (error, result) {
-            if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
+            if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(406, error.message));
            if (error) return next(new HttpError(500, error));

            res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -358,7 +404,7 @@ var authorization = [
    gServer.authorization({}, function (clientId, redirectURI, callback) {
        debug('authorization: client %s with callback to %s.', clientId, redirectURI);

-        clientdb.get(clientId, function (error, client) {
+        clients.get(clientId, function (error, client) {
            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
            if (error) return callback(error);

@@ -373,9 +419,12 @@ var authorization = [
        // Handle our different types of oauth clients
        var type = req.oauth2.client.type;

-        if (type === clientdb.TYPE_ADMIN) return next();
-        if (type === clientdb.TYPE_EXTERNAL) return next();
-        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');
+        if (type === clients.TYPE_ADMIN) {
+            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, 'admin'), { userId: req.oauth2.user.id });
+            return next();
+        }
+        if (type === clients.TYPE_EXTERNAL) return next();
+        if (type === clients.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');

        appdb.get(req.oauth2.client.appId, function (error, appObject) {
            if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
@@ -384,6 +433,8 @@ var authorization = [
                if (error) return sendError(req, res, 'Internal error');
                if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');

+                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id });
+
                next();
            });
        });
@@ -405,6 +456,31 @@ var token = [
    gServer.errorHandler()
 ];

+// tests if all requestedScopes are attached to the request
+function validateRequestedScopes(req, requestedScopes) {
+    assert.strictEqual(typeof req, 'object');
+    assert(Array.isArray(requestedScopes));
+
+    if (!req.authInfo || !req.authInfo.scope) return new Error('No scope found');
+
+    var scopes = req.authInfo.scope.split(',');
+
+    // check for roles separately
+    if (requestedScopes.indexOf(clients.SCOPE_ROLE_SDK) !== -1 && scopes.indexOf(clients.SCOPE_ROLE_SDK) === -1) {
+        return new Error('Missing required scope role "' + clients.SCOPE_ROLE_SDK + '"');
+    }
+
+    if (scopes.indexOf('*') !== -1) return null;
+
+    for (var i = 0; i < requestedScopes.length; ++i) {
+        if (scopes.indexOf(requestedScopes[i]) === -1) {
+            debug('scope: missing scope "%s".', requestedScopes[i]);
+            return new Error('Missing required scope "' + requestedScopes[i] + '"');
+        }
+    }
+
+    return null;
+}

 //  The scope middleware provides an auth middleware for routes.
 //
@@ -424,17 +500,8 @@ function scope(requestedScope) {
    return [
        passport.authenticate(['bearer'], { session: false }),
        function (req, res, next) {
-            if (!req.authInfo || !req.authInfo.scope) return next(new HttpError(401, 'No scope found'));
-            if (req.authInfo.scope === '*') return next();
-
-            var scopes = req.authInfo.scope.split(',');
-
-            for (var i = 0; i < requestedScopes.length; ++i) {
-                if (scopes.indexOf(requestedScopes[i]) === -1) {
-                    debug('scope: missing scope "%s".', requestedScopes[i]);
-                    return next(new HttpError(401, 'Missing required scope "' + requestedScopes[i] + '"'));
-                }
-            }
+            var error = validateRequestedScopes(req, requestedScopes);
+            if (error) return next(new HttpError(401, error.message));

            next();
        }
@@ -460,10 +527,12 @@ exports = module.exports = {
    passwordResetRequest: passwordResetRequest,
    passwordSentSite: passwordSentSite,
    passwordResetSite: passwordResetSite,
-    passwordSetupSite: passwordSetupSite,
    passwordReset: passwordReset,
+    accountSetupSite: accountSetupSite,
+    accountSetup: accountSetup,
    authorization: authorization,
    token: token,
+    validateRequestedScopes: validateRequestedScopes,
    scope: scope,
    csrf: csrf
 };
@@ -0,0 +1,86 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    update: update,
+    changePassword: changePassword,
+    setShowTutorial: setShowTutorial
+};
+
+var assert = require('assert'),
+    groups = require('../groups.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    user = require('../user.js'),
+    UserError = user.UserError,
+    _ = require('underscore');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, {
+            id: req.user.id,
+            username: req.user.username,
+            email: req.user.email,
+            admin: isAdmin,
+            displayName: req.user.displayName,
+            showTutorial: req.user.showTutorial
+        }));
+    });
+}
+
+function update(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
+
+    var data = _.pick(req.body, 'email', 'displayName');
+
+    user.update(req.user.id, data, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function changePassword(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'newPassword must be a string'));
+
+    user.setPassword(req.user.id, req.body.newPassword, function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setShowTutorial(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.showTutorial !== 'boolean') return next(new HttpError(400, 'showTutorial must be a boolean.'));
+
+    user.setShowTutorial(req.user.id, req.body.showTutorial, function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -18,6 +16,9 @@ exports = module.exports = {
    getBackupConfig: getBackupConfig,
    setBackupConfig: setBackupConfig,

+    getTimeZone: getTimeZone,
+    setTimeZone: setTimeZone,
+
    setCertificate: setCertificate,
    setAdminCertificate: setAdminCertificate
 };
@@ -45,7 +46,7 @@ function setAutoupdatePattern(req, res, next) {
    if (typeof req.body.pattern !== 'string') return next(new HttpError(400, 'pattern is required'));

    settings.setAutoupdatePattern(req.body.pattern, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid pattern'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error) return next(new HttpError(500, error));

        next(new HttpSuccess(200));
@@ -58,8 +59,9 @@ function setCloudronName(req, res, next) {
    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name is required'));

    settings.setCloudronName(req.body.name, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid name'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
        if (error) return next(new HttpError(500, error));
+
        next(new HttpSuccess(200));
    });
 }
@@ -67,10 +69,32 @@ function setCloudronName(req, res, next) {
 function getCloudronName(req, res, next) {
    settings.getCloudronName(function (error, name) {
        if (error) return next(new HttpError(500, error));
+
        next(new HttpSuccess(200, { name: name }));
    });
 }

+function getTimeZone(req, res, next) {
+    settings.getTimeZone(function (error, tz) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { timeZone: tz }));
+    });
+}
+
+function setTimeZone(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.timeZone !== 'string') return next(new HttpError(400, 'timeZone is required'));
+
+    settings.setTimeZone(req.body.timeZone, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
 function setCloudronAvatar(req, res, next) {
    assert.strictEqual(typeof req.files, 'object');

@@ -79,6 +103,7 @@ function setCloudronAvatar(req, res, next) {

    settings.setCloudronAvatar(avatar, function (error) {
        if (error) return next(new HttpError(500, error));
+
        next(new HttpSuccess(202, {}));
    });
 }
@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';

 exports = module.exports = {
@@ -8,9 +6,11 @@ exports = module.exports = {
    retire: retire
 };

-var cloudron = require('../cloudron.js'),
+var backups = require('../backups.js'),
+    BackupsError = require('../backups.js').BackupsError,
+    cloudron = require('../cloudron.js'),
    CloudronError = require('../cloudron.js').CloudronError,
-    debug = require('debug')('box:routes/internal'),
+    debug = require('debug')('box:routes/sysadmin'),
    HttpError = require('connect-lastmile').HttpError,
    HttpSuccess = require('connect-lastmile').HttpSuccess;

@@ -19,8 +19,9 @@ function backup(req, res, next) {

    // note that cloudron.backup only waits for backup initiation and not for backup to complete
    // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    var auditSource = { userId: null, username: 'sysadmin' };
+    backups.backup(auditSource, function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
        if (error) return next(new HttpError(500, error));

        next(new HttpSuccess(202, {}));
@@ -31,7 +32,8 @@ function update(req, res, next) {
    debug('triggering update');

    // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    var auditSource = { userId: null, username: 'sysadmin' };
+    cloudron.updateToLatest(auditSource, function (error) {
        if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
        if (error) return next(new HttpError(500, error));
@@ -12,7 +12,7 @@ var appdb = require('../../appdb.js'),
    path = require('path'),
    async = require('async'),
    child_process = require('child_process'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
    config = require('../../config.js'),
    constants = require('../../constants.js'),
    database = require('../../database.js'),
@@ -23,18 +23,18 @@ var appdb = require('../../appdb.js'),
    http = require('http'),
    https = require('https'),
    js2xml = require('js2xmlparser'),
+    ldap = require('../../ldap.js'),
    net = require('net'),
    nock = require('nock'),
    paths = require('../../paths.js'),
-    redis = require('redis'),
-    superagent = require('superagent'),
    safe = require('safetydance'),
    server = require('../../server.js'),
    settings = require('../../settings.js'),
+    simpleauth = require('../../simpleauth.js'),
+    superagent = require('superagent'),
    taskmanager = require('../../taskmanager.js'),
    tokendb = require('../../tokendb.js'),
    url = require('url'),
-    util = require('util'),
    uuid = require('node-uuid'),
    _ = require('underscore');

@@ -42,9 +42,9 @@ var SERVER_URL = 'http://localhost:' + config.get('port');

 // Test image information
 var TEST_IMAGE_REPO = 'cloudron/test';
-var TEST_IMAGE_TAG = '10.0.0';
+var TEST_IMAGE_TAG = '15.0.0';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
-var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
+// var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();

 var APP_STORE_ID = 'test', APP_ID;
 var APP_LOCATION = 'appslocation';
@@ -58,8 +58,8 @@ var APP_MANIFEST_1 = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-a
 APP_MANIFEST_1.dockerImage = TEST_IMAGE;
 APP_MANIFEST_1.singleUser = true;

-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
-var USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
+var USER_1_ID = null, USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;

@@ -112,48 +112,21 @@ describe('Apps', function () {
    var imageCreated = false;

    before(function (done) {
-        console.log('Starting addons, this can take 10 seconds');
-
-        child_process.exec(__dirname + '/start_addons.sh', function (error) {
-            if (error) return done(error);
-
-            dockerProxy = startDockerProxy(function interceptor(req, res) {
-                if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
-                    imageCreated = true;
-                    res.writeHead(200);
-                    res.end();
-                    return true;
-                } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
-                    imageDeleted = true;
-                    res.writeHead(200);
-                    res.end();
-                    return true;
-                }
-                return false;
-            }, done);
-        });
-    });
-
-    after(function (done) {
-        // child_process.exec('docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb; docker rm -f mail', function (error) {
-        child_process.exec('docker ps | awk \'{print $1}\' | xargs docker rm -f', function () {
-            dockerProxy.close(done);
-        });
-    });
-
-
-    /*
-        Individual sub category setup and cleanup
-    */
-    function setup(done) {
        config._reset();

+        process.env.CREATE_INFRA = 1;
+
+        safe.fs.unlinkSync(paths.INFRA_VERSION_FILE);
+        child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+
        async.series([
            // first clear, then start server. otherwise, taskmanager spins up tasks for obsolete appIds
            database.initialize,
            database._clear,

            server.start.bind(server),
+            ldap.start,
+            simpleauth.start,

            function (callback) {
                var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
@@ -182,6 +155,8 @@ describe('Apps', function () {
                       .end(function (err, res) {
                    expect(res.statusCode).to.equal(201);

+                    USER_1_ID = res.body.id;
+
                    callback(null);
                });
            },
@@ -190,58 +165,104 @@ describe('Apps', function () {
                token_1 = tokendb.generateToken();

                // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+            },
+
+            function (callback) {
+                dockerProxy = startDockerProxy(function interceptor(req, res) {
+                    if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
+                        imageCreated = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
+                        imageDeleted = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    }
+                    return false;
+                }, callback);
            },

            settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
            settings.setTlsConfig.bind(null, { provider: 'caas' }),
            settings.setBackupConfig.bind(null, { provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
-        ], done);
-    }
+        ], function (error) {
+            if (error) return done(error);
+
+            console.log('This test can take ~30 seconds to start as it waits for infra to be ready');
+            setTimeout(done, 30000);
+        });
+    });
+
+    after(function (done) {
+        delete process.env.CREATE_INFRA;
+
+        // child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+        dockerProxy.close(function () { });

-    function cleanup(done) {
        // db is not cleaned up here since it's too late to call it after server.stop. if called before server.stop taskmanager apptasks are unhappy :/
        async.series([
            taskmanager.stopPendingTasks,
            taskmanager.waitForPendingTasks,
            server.stop,
+            ldap.stop,
+            simpleauth.stop,
            config._reset,
        ], done);
-    }
+    });

    describe('App API', function () {
-        this.timeout(50000);
-
-        before(setup);
-
        after(function (done) {
-            APP_ID = null;
-            cleanup(done);
+            appdb._clear(done); // TODO: test proper uninstall (requires mock for aws)
        });

        it('app install fails - missing manifest', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD })
+                   .send({ password: PASSWORD })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('manifest is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                done();
            });
        });

-        it('app install fails - missing appId', function (done) {
+        it('app install fails - null manifest', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ manifest: APP_MANIFEST, password: PASSWORD })
+                   .send({ manifest: null, password: PASSWORD })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('appStoreId is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                done();
            });
        });

-        it('app install fails - invalid json', function (done) {
+        it('app install fails - bad manifest format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: 'epic', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('manifest must be an object');
+                done();
+            });
+        });
+
+        it('app install fails - empty appStoreId format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: null, appStoreId: '', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
+                done();
+            });
+        });
+
+       it('app install fails - invalid json', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
                   .send('garbage')
@@ -254,7 +275,7 @@ describe('Apps', function () {
        it('app install fails - invalid location', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('Hostname can only contain alphanumerics and hyphen');
@@ -265,7 +286,7 @@ describe('Apps', function () {
        it('app install fails - invalid location type', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('location is required');
@@ -276,7 +297,7 @@ describe('Apps', function () {
        it('app install fails - reserved admin location', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
@@ -287,7 +308,7 @@ describe('Apps', function () {
        it('app install fails - reserved api location', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql(constants.API_LOCATION + ' is reserved');
@@ -298,7 +319,7 @@ describe('Apps', function () {
        it('app install fails - portBindings must be object', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('portBindings must be an object');
@@ -309,7 +330,7 @@ describe('Apps', function () {
        it('app install fails - accessRestriction is required', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('accessRestriction is required');
@@ -320,7 +341,7 @@ describe('Apps', function () {
        it('app install fails - accessRestriction type is wrong', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('accessRestriction is required');
@@ -331,7 +352,7 @@ describe('Apps', function () {
        it('app install fails - accessRestriction no users not allowed', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -342,7 +363,7 @@ describe('Apps', function () {
        it('app install fails - accessRestriction too many users not allowed', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -353,50 +374,64 @@ describe('Apps', function () {
        it('app install fails for non admin', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token_1 })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(403);
                done();
            });
        });

-        it('app install fails due to purchase failure', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+        it('app install fails because manifest download fails', function (done) {
+            var fake = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(404, {});

            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(fake.isDone()).to.be.ok();
+                done();
+            });
+        });
+
+        it('app install fails due to purchase failure', function (done) {
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(402);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                done();
            });
        });

        it('app install succeeds with purchase', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});

            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(202);
                expect(res.body.id).to.be.a('string');
                APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                done();
            });
        });

        it('app install fails because of conflicting location', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(409);
-                expect(fake.isDone()).to.be.ok();
                done();
            });
        });
@@ -444,24 +479,6 @@ describe('Apps', function () {
             });
        });

-        it('can get appBySubdomain', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/subdomains/' + APP_LOCATION)
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(200);
-                expect(res.body.id).to.eql(APP_ID);
-                expect(res.body.installationState).to.be.ok();
-                done();
-            });
-        });
-
-        it('cannot get invalid app by Subdomain', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/subdomains/tikaloma')
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(404);
-                done();
-            });
-        });
-
        it('cannot uninstall invalid app', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/whatever/uninstall')
                .send({ password: PASSWORD })
@@ -512,23 +529,23 @@ describe('Apps', function () {
        });

        it('app install succeeds already purchased', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});

            superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                expect(res.statusCode).to.equal(202);
                expect(res.body.id).to.be.a('string');
                APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                done();
            });
        });

        it('app install succeeds without password but developer token', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
            settings.setDeveloperMode(true, function (error) {
                expect(error).to.be(null);

@@ -537,7 +554,7 @@ describe('Apps', function () {
                       .end(function (error, result) {
                    expect(error).to.not.be.ok();
                    expect(result.statusCode).to.equal(200);
-                    expect(result.body.expiresAt).to.be.a('number');
+                    expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                    expect(result.body.token).to.be.a('string');

                    // overwrite non dev token
@@ -545,11 +562,10 @@ describe('Apps', function () {

                    superagent.post(SERVER_URL + '/api/v1/apps/install')
                           .query({ access_token: token })
-                           .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
+                           .send({ manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
                           .end(function (err, res) {
                        expect(res.statusCode).to.equal(202);
                        expect(res.body.id).to.be.a('string');
-                        expect(fake.isDone()).to.be.ok();
                        APP_ID = res.body.id;
                        done();
                    });
@@ -580,8 +596,6 @@ describe('Apps', function () {
            imageCreated = false;

            async.series([
-                setup,
-
                function (callback) {
                    apiHockInstance
                        .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
@@ -610,7 +624,6 @@ describe('Apps', function () {
            APP_ID = null;

            async.series([
-                cleanup,
                apiHockServer.close.bind(apiHockServer),
                awsHockServer.close.bind(awsHockServer)
            ], done);
@@ -619,7 +632,8 @@ describe('Apps', function () {
        var appResult = null /* the json response */, appEntry = null /* entry from database */;

        it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});

            var count = 0;
            function checkInstallStatus() {
@@ -636,12 +650,13 @@ describe('Apps', function () {

            superagent.post(SERVER_URL + '/api/v1/apps/install')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                expect(res.body.id).to.be.a('string');
-                expect(res.body.id).to.be.eql(APP_ID);
+                APP_ID = res.body.id;
                checkInstallStatus();
            });
        });
@@ -671,14 +686,7 @@ describe('Apps', function () {
                expect(data.Config.Env).to.contain('APP_ORIGIN=https://' + config.appFqdn(APP_LOCATION));
                expect(data.Config.Env).to.contain('APP_DOMAIN=' + config.appFqdn(APP_LOCATION));
                expect(data.Config.Hostname).to.be(APP_LOCATION);
-                clientdb.getByAppIdAndType(appResult.id, clientdb.TYPE_OAUTH, function (error, client) {
-                    expect(error).to.not.be.ok();
-                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
-                    expect(client.clientSecret.length).to.be(64); // 32 hex chars (256 bits)
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
-                    done();
-                });
+                done();
            });
        });

@@ -724,6 +732,65 @@ describe('Apps', function () {
            });
        });

+        it('installation - app responnds to http request', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort).end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.status).to.be('OK');
+                done();
+            });
+        });
+
+        it('installation - oauth addon config', function (done) {
+            var appContainer = docker.getContainer(appEntry.containerId);
+            appContainer.inspect(function (error, data) {
+                expect(error).to.not.be.ok();
+
+                clients.getByAppIdAndType(APP_ID, clients.TYPE_OAUTH, function (error, client) {
+                    expect(error).to.not.be.ok();
+                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
+                    expect(client.clientSecret.length).to.be(64); // 32 hex chars (256 bits)
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
+                    done();
+                });
+            });
+        });
+
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
+                done();
+            });
+        });
+
+        it('installation - app can check addons', function (done) {
+            console.log('This test can take a while as it waits for scheduler addon to tick');
+
+            async.retry({ times: 15, interval: 6000 }, function (callback) {
+                superagent.get('http://localhost:' + appEntry.httpPort + '/check_addons')
+                    .query({ username: USERNAME, password: PASSWORD })
+                    .end(function (err, res) {
+
+                    expect(!err).to.be.ok();
+                    expect(res.statusCode).to.equal(200);
+
+                    delete res.body.recvmail; // unclear why dovecot mail delivery won't work
+                    delete res.body.stdenv; // cannot access APP_ORIGIN
+
+                    for (var key in res.body) {
+                        if (res.body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(res.body));
+                    }
+
+                    callback();
+                });
+            }, done);
+        });
+
        var redisIp, exportedRedisPort;

        it('installation - redis addon created', function (done) {
@@ -741,101 +808,6 @@ describe('Apps', function () {
            });
        });

-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('installation - mysql addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var mysqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('MYSQL_URL=') === 0) mysqlUrl = env.split('=')[1]; });
-                expect(mysqlUrl).to.be.ok();
-
-                var urlp = url.parse(mysqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('MYSQL_PORT=3306');
-                expect(data.Config.Env).to.contain('MYSQL_HOST=mysql');
-                expect(data.Config.Env).to.contain('MYSQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('MYSQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('MYSQL_DATABASE=' + dbname);
-
-                var cmd = util.format('mysql -h %s -u%s -p%s --database=%s -e "CREATE TABLE IF NOT EXISTS foo (id INT);"',
-                    'mysql', username, password, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    // expect(stderr.length).to.be(0); // "Warning: Using a password on the command line interface can be insecure."
-                    done();
-                });
-            });
-        });
-
-        it('installation - postgresql addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var postgresqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('POSTGRESQL_URL=') === 0) postgresqlUrl = env.split('=')[1]; });
-                expect(postgresqlUrl).to.be.ok();
-
-                var urlp = url.parse(postgresqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('POSTGRESQL_PORT=5432');
-                expect(data.Config.Env).to.contain('POSTGRESQL_HOST=postgresql');
-                expect(data.Config.Env).to.contain('POSTGRESQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('POSTGRESQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('POSTGRESQL_DATABASE=' + dbname);
-
-                var cmd = util.format('bash -c "PGPASSWORD=%s psql -q -h %s -U%s --dbname=%s -e \'CREATE TABLE IF NOT EXISTS foo (id INT);\'"',
-                    password, 'postgresql', username, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    expect(stderr.length).to.be(0);
-                    done();
-                });
-            });
-        });
-
-        it('installation - scheduler', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (retryCallback) {
-                if (fs.existsSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env')) return retryCallback();
-
-                retryCallback(new Error('not run yet'));
-            }, done);
-        });
-
        xit('logs - stdout and stderr', function (done) {
            superagent.get(SERVER_URL + '/api/v1/apps/' + APP_ID + '/logs')
                .query({ access_token: token })
@@ -955,6 +927,27 @@ describe('Apps', function () {
            checkStartState();
        });

+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            async.retry({ times: 15, interval: 6000 }, function (callback) {
+                superagent.get('http://localhost:' + appEntry.httpPort + '/check_addons')
+                    .query({ username: USERNAME, password: PASSWORD })
+                    .end(function (err, res) {
+                    expect(!err).to.be.ok();
+                    expect(res.statusCode).to.equal(200);
+
+                    delete res.body.recvmail; // unclear why dovecot mail delivery won't work
+                    delete res.body.stdenv; // cannot access APP_ORIGIN
+
+                    for (var key in res.body) {
+                        if (res.body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(res.body));
+                    }
+
+                    callback();
+                });
+            }, done);
+        });
+
        it('can uninstall app', function (done) {
            var count = 0;
            function checkUninstallStatus() {
@@ -1038,8 +1031,6 @@ describe('Apps', function () {
            APP_ID = uuid.v4();

            async.series([
-                setup,
-
                function (callback) {
                    config.set('fqdn', 'test.foobar.com');
                    callback();
@@ -1076,7 +1067,6 @@ describe('Apps', function () {
        after(function (done) {
            APP_ID = null;
            async.series([
-                cleanup,
                apiHockServer.close.bind(apiHockServer),
                awsHockServer.close.bind(awsHockServer)
            ], done);
@@ -1085,7 +1075,8 @@ describe('Apps', function () {
        var appResult = null, appEntry = null;

        it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});

            var count = 0;
            function checkInstallStatus() {
@@ -1102,11 +1093,12 @@ describe('Apps', function () {

            superagent.post(SERVER_URL + '/api/v1/apps/install')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
-                expect(res.body.id).to.equal(APP_ID);
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
+                APP_ID = res.body.id;
                checkInstallStatus();
            });
        });
@@ -1194,6 +1186,39 @@ describe('Apps', function () {
            });
        });

+
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
+                done();
+            });
+        });
+
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            async.retry({ times: 15, interval: 6000 }, function (callback) {
+                superagent.get('http://localhost:' + appEntry.httpPort + '/check_addons')
+                    .query({ username: USERNAME, password: PASSWORD })
+                    .end(function (err, res) {
+                    expect(!err).to.be.ok();
+                    expect(res.statusCode).to.equal(200);
+
+                    delete res.body.recvmail; // unclear why dovecot mail delivery won't work
+                    delete res.body.stdenv; // cannot access APP_ORIGIN
+
+                    for (var key in res.body) {
+                        if (res.body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(res.body));
+                    }
+
+                    callback();
+                });
+            }, done);
+        });
+
        var redisIp, exportedRedisPort;

        it('installation - redis addon created', function (done) {
@@ -1211,37 +1236,6 @@ describe('Apps', function () {
            });
        });

-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                function checkRedis() {
-                    var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                    client.on('error', done);
-                    client.set('key', 'value');
-                    client.get('key', function (err, reply) {
-                        expect(err).to.not.be.ok();
-                        expect(reply.toString()).to.be('value');
-                        client.end();
-                        done();
-                    });
-                }
-
-                setTimeout(checkRedis, 1000); // the bridge network takes time to come up?
-            });
-        });
-
        function checkConfigureStatus(count, done) {
            assert.strictEqual(typeof count, 'number');
            assert.strictEqual(typeof done, 'function');
@@ -1257,20 +1251,20 @@ describe('Apps', function () {
            });
        }

-        it('cannot reconfigure app with missing location', function (done) {
+        it('cannot reconfigure app with bad location', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: 1234, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                done();
            });
        });

-        it('cannot reconfigure app with missing accessRestriction', function (done) {
+        it('cannot reconfigure app with bad accessRestriction', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
+                  .send({ password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: false })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                done();
@@ -1280,7 +1274,7 @@ describe('Apps', function () {
        it('cannot reconfigure app with only the cert, no key', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                done();
@@ -1290,27 +1284,27 @@ describe('Apps', function () {
        it('cannot reconfigure app with only the key, no cert', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, key: validKey1 })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                done();
            });
        });

-        it('cannot reconfigure app with cert not bein a string', function (done) {
+        it('cannot reconfigure app with cert not being a string', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                done();
            });
        });

-        it('cannot reconfigure app with key not bein a string', function (done) {
+        it('cannot reconfigure app with key not being a string', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: 1234 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, cert: validCert1, key: 1234 })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(400);
                done();
@@ -1320,7 +1314,7 @@ describe('Apps', function () {
        it('non admin cannot reconfigure app', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token_1 })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(403);
                done();
@@ -1330,7 +1324,7 @@ describe('Apps', function () {
        it('can reconfigure app', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(202);
                checkConfigureStatus(0, done);
@@ -1374,47 +1368,31 @@ describe('Apps', function () {
            });
        });

-        it('redis addon works after reconfiguration', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            async.retry({ times: 15, interval: 6000 }, function (callback) {
+                superagent.get('http://localhost:' + appEntry.httpPort + '/check_addons')
+                    .query({ username: USERNAME, password: PASSWORD })
+                    .end(function (err, res) {
+                    expect(!err).to.be.ok();
+                    expect(res.statusCode).to.equal(200);

-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
+                    delete res.body.recvmail; // unclear why dovecot mail delivery won't work
+                    delete res.body.stdenv; // cannot access APP_ORIGIN

-                expect(urlp.hostname).to.be('redis-' + APP_ID);
+                    for (var key in res.body) {
+                        if (res.body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(res.body));
+                    }

-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
+                    callback();
                });
-            });
-        });
-
-        it('scheduler works after reconfiguration', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (callback) {
-                var data = safe.fs.readFileSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env', 'utf8');
-
-                if (data && data.indexOf('ECHO_SERVER_PORT=7172') !== -1) return callback();
-
-                callback(new Error('not run yet'));
            }, done);
        });

        it('can reconfigure app with custom certificate', function (done) {
            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                  .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
                  .end(function (err, res) {
                expect(res.statusCode).to.equal(202);
                checkConfigureStatus(0, done);
@@ -19,11 +19,13 @@ var appdb = require('../../appdb.js'),

 var SERVER_URL = 'http://localhost:' + config.get('port');

-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;

 var server;
 function setup(done) {
+    config.setVersion('1.2.3');
+
    async.series([
        server.start.bind(server),

@@ -51,7 +53,7 @@ function setup(done) {

        function addApp(callback) {
            var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
        },

        function createSettings(callback) {
@@ -72,35 +74,6 @@ describe('Backups API', function () {
    before(setup);
    after(cleanup);

-    describe('get', function () {
-        it('cannot get backups with appstore superagent failing', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(401, {});
-
-            superagent.get(SERVER_URL + '/api/v1/backups')
-                   .query({ access_token: token })
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(503);
-                expect(req.isDone()).to.be.ok();
-                done();
-            });
-        });
-
-        it('can get backups', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(200, { backups: ['foo', 'bar']});
-
-            superagent.get(SERVER_URL + '/api/v1/backups')
-                   .query({ access_token: token })
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(200);
-                expect(req.isDone()).to.be.ok();
-                expect(res.body.backups).to.be.an(Array);
-                expect(res.body.backups[0]).to.eql('foo');
-                expect(res.body.backups[1]).to.eql('bar');
-                done();
-            });
-        });
-    });
-
    describe('create', function () {
        it('fails due to mising token', function (done) {
            superagent.post(SERVER_URL + '/api/v1/backups')
@@ -122,7 +95,7 @@ describe('Backups API', function () {
        it('succeeds', function (done) {
            var scope = nock(config.apiServerOrigin())
                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
-                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey' } });

            superagent.post(SERVER_URL + '/api/v1/backups')
                   .query({ access_token: token })
@@ -8,6 +8,7 @@

 var async = require('async'),
    config = require('../../config.js'),
+    clients = require('../../clients.js'),
    database = require('../../database.js'),
    oauth2 = require('../oauth2.js'),
    expect = require('expect.js'),
@@ -20,7 +21,7 @@ var async = require('async'),

 var SERVER_URL = 'http://localhost:' + config.get('port');

-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token

 function cleanup(done) {
@@ -173,6 +174,8 @@ describe('OAuth Clients API', function () {
                    expect(result.body.redirectURI).to.be.a('string');
                    expect(result.body.clientSecret).to.be.a('string');
                    expect(result.body.scope).to.be.a('string');
+                    expect(result.body.type).to.equal(clients.TYPE_EXTERNAL);
+
                    done();
                });
            });
@@ -412,7 +415,7 @@ describe('Clients', function () {
            server.start.bind(server),
            database._clear.bind(null),
            function (callback) {
-            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
                var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});

                superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
@@ -427,7 +430,16 @@ describe('Clients', function () {
                    // stash for further use
                    token = result.body.token;

-                    callback();
+                    superagent.get(SERVER_URL + '/api/v1/profile')
+                            .query({ access_token: token })
+                            .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(result.statusCode).to.eql(200);
+
+                        USER_0.id = result.body.id;
+
+                        callback();
+                    });
                });
            }
        ], done);
@@ -531,7 +543,7 @@ describe('Clients', function () {
                expect(result.statusCode).to.equal(200);

                expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.username);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);

                done();
            });
@@ -584,7 +596,7 @@ describe('Clients', function () {
                expect(result.statusCode).to.equal(200);

                expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.username);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);

                superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin/tokens')
                .query({ access_token: token })
@@ -18,7 +18,7 @@ var async = require('async'),

 var SERVER_URL = 'http://localhost:' + config.get('port');

-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token

 var server;
@@ -233,7 +233,9 @@ describe('Cloudron', function () {
        });

        it('succeeds', function (done) {
-            var scope = nock(config.apiServerOrigin()).get('/api/v1/boxes/localhost?token=' + config.token()).reply(200, { box: { region: 'sfo', size: '1gb' }});
+            var scope = nock(config.apiServerOrigin())
+                  .get('/api/v1/boxes/localhost?token=' + config.token())
+                  .reply(200, { box: { region: 'sfo', size: '1gb' }, user: { }});

            superagent.get(SERVER_URL + '/api/v1/cloudron/config')
                   .query({ access_token: token })
@@ -341,7 +343,7 @@ describe('Cloudron', function () {

        it('succeeds with app type', function (done) {
            superagent.post(SERVER_URL + '/api/v1/cloudron/feedback')
-                   .send({ type: 'app', subject: 'some subject', description: 'some description' })
+                   .send({ type: 'app_missing', subject: 'some subject', description: 'some description' })
                   .query({ access_token: token })
                   .end(function (error, result) {
                expect(result.statusCode).to.equal(201);
@@ -17,7 +17,7 @@ var async = require('async'),

 var SERVER_URL = 'http://localhost:' + config.get('port');

-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token

 var server;
@@ -297,7 +297,16 @@ describe('Developer API', function () {

        it('fails with unknown username', function (done) {
            superagent.post(SERVER_URL + '/api/v1/developer/login')
-                   .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                   .send({ username: USERNAME + USERNAME, password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails with unknown email', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: USERNAME + EMAIL, password: PASSWORD })
                   .end(function (error, result) {
                expect(result.statusCode).to.equal(401);
                done();
@@ -318,7 +327,18 @@ describe('Developer API', function () {
                   .send({ username: USERNAME, password: PASSWORD })
                   .end(function (error, result) {
                expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                expect(result.body.token).to.be.a('string');
+                done();
+            });
+        });
+
+        it('with uppercase username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                expect(result.body.token).to.be.a('string');
                done();
            });
@@ -329,10 +349,77 @@ describe('Developer API', function () {
                   .send({ username: EMAIL, password: PASSWORD })
                   .end(function (error, result) {
                expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                expect(result.body.token).to.be.a('string');
+                done();
+            });
+        });
+
+        it('with uppercase email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                expect(result.body.token).to.be.a('string');
                done();
            });
        });
    });
+
+    describe('sdk tokens are valid without password checks', function () {
+        var token_normal, token_sdk;
+
+        before(function (done) {
+            async.series([
+                setup,
+
+                settings.setDeveloperMode.bind(null, true),
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        token_normal = result.body.token;
+
+                        superagent.post(SERVER_URL + '/api/v1/developer/login')
+                          .send({ username: USERNAME, password: PASSWORD })
+                          .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+                            expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                            expect(result.body.token).to.be.a('string');
+
+                            token_sdk = result.body.token;
+
+                            callback();
+                        });
+                    });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails with non sdk token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_normal }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_sdk }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+    });
 });
@@ -0,0 +1,152 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    nock = require('nock'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    tokendb = require('../../tokendb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var USER_1_ID = null, token_1;
+
+function setup(done) {
+    config.setVersion('1.2.3');
+
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        },
+
+        function (callback) {
+            superagent.post(SERVER_URL + '/api/v1/users')
+                   .query({ access_token: token })
+                   .send({ username: 'nonadmin', email: 'notadmin@server.test', invite: false })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(201);
+
+                USER_1_ID = res.body.id;
+
+                callback(null);
+            });
+        },
+
+        function (callback) {
+            token_1 = tokendb.generateToken();
+
+            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+            tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+        }
+
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Eventlog API', function () {
+    before(setup);
+    after(cleanup);
+
+    describe('get', function () {
+        it('fails due to wrong token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token.toUpperCase() })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails for non-admin', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token_1, page: 1, per_page: 10 })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(403);
+
+                done();
+            });
+        });
+
+        it('succeeds for admin', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10 })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length >= 2).to.be.ok(); // activate, user.add
+
+                done();
+            });
+        });
+
+        it('succeeds with action', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10, action: 'cloudron.activate' })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length).to.equal(1);
+
+                done();
+            });
+        });
+
+        it('succeeds with search', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10, search: EMAIL })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length).to.equal(1);
+
+                done();
+            });
+        });
+
+        it('succeeds with search', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/eventlog')
+                   .query({ access_token: token, page: 1, per_page: 10, search: EMAIL, action: 'cloudron.activate' })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.eventlogs.length).to.equal(0);
+
+                done();
+            });
+        });
+    });
+});
@@ -6,24 +6,22 @@

 'use strict';

-var appdb = require('../../appdb.js'),
-    async = require('async'),
+var async = require('async'),
    config = require('../../config.js'),
    database = require('../../database.js'),
    expect = require('expect.js'),
    groups = require('../../groups.js'),
    superagent = require('superagent'),
    server = require('../../server.js'),
-    settings = require('../../settings.js'),
    tokendb = require('../../tokendb.js'),
-    nock = require('nock'),
-    userdb = require('../../userdb.js');
+    nock = require('nock');

 var SERVER_URL = 'http://localhost:' + config.get('port');

-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1337', EMAIL_1 ='happy@me.com';
 var token, token_1 = null;
+var userId, userId_1 = null;

 var server;
 function setup(done) {
@@ -48,7 +46,16 @@ function setup(done) {
                // stash token for further use
                token = result.body.token;

-                callback();
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                      .query({ access_token: token })
+                      .end(function (error, result) {
+                    expect(result).to.be.ok();
+                    expect(result.statusCode).to.eql(200);
+
+                    userId = result.body.id;
+
+                    callback();
+                });
            });
        },
        function (callback) {
@@ -60,9 +67,10 @@ function setup(done) {
                expect(result.statusCode).to.eql(201);

                token_1 = tokendb.generateToken();
+                userId_1 = result.body.id;

                // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, userId_1, 'test-client-id',  Date.now() + 100000, '*', callback);
            });
        }
  ], done);
@@ -106,6 +114,9 @@ describe('Groups API', function () {
                expect(res.body.groups).to.be.an(Array);
                expect(res.body.groups.length).to.be(1);
                expect(res.body.groups[0].name).to.eql('admin');
+                expect(res.body.groups[0].userIds).to.be.an(Array);
+                expect(res.body.groups[0].userIds.length).to.be(1);
+                expect(res.body.groups[0].userIds[0]).to.be(userId);
                done();
            });
        });
@@ -168,7 +179,7 @@ describe('Groups API', function () {
                expect(result.statusCode).to.equal(200);
                expect(result.body.name).to.be('admin');
                expect(result.body.userIds.length).to.be(1);
-                expect(result.body.userIds[0]).to.be(USERNAME);
+                expect(result.body.userIds[0]).to.be(userId);
                done();
            });
        });
@@ -213,7 +224,7 @@ describe('Groups API', function () {
        });

        it('cannot add user to invalid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                  .query({ access_token: token })
                  .send({ groupIds: [ 'admin', 'something' ]})
                  .end(function (error, result) {
@@ -223,7 +234,7 @@ describe('Groups API', function () {
        });

        it('can add user to valid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                  .query({ access_token: token })
                  .send({ groupIds: [ 'admin', 'group0', 'group1' ]})
                  .end(function (error, result) {
@@ -232,8 +243,8 @@ describe('Groups API', function () {
            });
        });

-        it('can remove last user from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+        it('cannot remove self from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                  .query({ access_token: token })
                  .send({ groupIds: [ 'group0', 'group1' ]})
                  .end(function (error, result) {
@@ -241,5 +252,37 @@ describe('Groups API', function () {
                done();
            });
        });
+
+        it('can add another user to admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId_1 + '/groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'admin' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('lists members of admin group', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/admin')
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.userIds.length).to.be(2);
+                expect(result.body.userIds[0]).to.be(userId);
+                expect(result.body.userIds[1]).to.be(userId_1);
+                done();
+            });
+        });
+
+        it('remove activation user from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+                  .query({ access_token: token_1 })
+                  .send({ groupIds: [ 'group0', 'group1' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204); // user_1 is still admin, so we can remove the other person
+                done();
+            });
+        });
    });
 });
@@ -0,0 +1,211 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    nock = require('nock'),
+    userdb = require('../../userdb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var MAILBOX_ID = 'mailbox';
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.set('fqdn', 'foobar.com');
+
+    async.series([
+        server.start.bind(server),
+
+        userdb._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Mailbox API', function () {
+    this.timeout(10000);
+
+    before(setup);
+    after(cleanup);
+
+    it('cannot create a mailbox without name param', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot create a mailbox without token', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(401);
+            done();
+        });
+    });
+
+    it('cannot create invalid mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: 'no-reply' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can create mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('can get mailbox', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.name).to.equal(MAILBOX_ID);
+            expect(res.body.creationTime).to.be.ok();
+            done();
+        });
+    });
+
+    it('cannot set with invalid alias', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'a' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot set with invalid type', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'apple', 34 ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can set aliases of mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'alias1', 'alias2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            done();
+        });
+    });
+
+    it('can list mailboxes', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.mailboxes).to.be.an(Array);
+            expect(res.body.mailboxes[0].name).to.be(MAILBOX_ID);
+            done();
+        });
+    });
+
+    it('can get aliases', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.aliases).to.be.an(Array);
+            expect(res.body.aliases[0]).to.be('alias1');
+            expect(res.body.aliases[1]).to.be('alias2');
+            done();
+        });
+    });
+
+    it('can add another mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID + '2' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('cannot alias existing mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ MAILBOX_ID + '2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(409);
+            done();
+        });
+    });
+
+    it('can delete mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(204);
+            done();
+        });
+    });
+
+    it('cannot delete random mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(404);
+            done();
+        });
+    });
+});
--- a/Show More
+++ b/Show More