Pass manifest to backupApp

we were passing the current manifest to the backup code which meant that
the app version and manifest was incorrect.

CHANGES

@@ -400,3 +400,138 @@
 - Improved box update management using prereleases
 - Less aggressive disk space checks
 
+[0.8.0]
+- MySQL addon : multiple database support
+
+[0.8.1]
+- Set Host HTTP header when querying healthCheckPath
+- Show application Changelog in app update emails
+
+[0.9.0]
+- Fix bug in multdb mysql addon backup
+- Add initial user group support
+- Improved app memory limit handling
+
+[0.9.1]
+- Introduce per app group access control
+
+[0.9.2]
+- Fix bug where reconfiguring apps would trigger memory limit warning
+- Allow more apps to be installed in bigger sized cloudrons
+- Allow user to override memory limit warning and install anyway
+
+[0.9.3]
+- Admin flag is handled outside of groups
+- User interface fixes for groups
+- Allow to set access restrictions on app installation
+
+[0.10.0]
+- Upgrade to docker 1.10.2
+- Fix MySQL addon to handle heavier loads
+- Allow listing and download of backups (using the CLI tool)
+- Ubuntu security updates till 8th March 2016 (http://www.ubuntu.com/usn)
+
+[0.10.1]
+- Fix Let's Encrypt certificate renewal
+
+[0.10.2]
+- Apps can now bind with username or email with LDAP
+- Disallow updating an app with mismatching manifest id
+- Use admin domain instead of naked domain in the SPF record
+- Download Lets Encrypt intermediate cert
+
+[0.10.3]
+- Store the backup config for each backup. This will allow using multiple buckets/providers for backups simultaneously.
+- Fix SPF record check
+
+[0.10.4]
+- Fix restore for droplets in EU region
+
+[0.11.0]
+- Store backups in the same region as the Cloudron
+- Fix PCRE security issue (http://www.ubuntu.com/usn/usn-2943-1/)
+
+[0.11.1]
+- Improve the backup logic
+
+[0.11.2]
+- Allow users to choose a username on first sign up
+- Fix app graphs
+
+[0.12.0]
+- Fix upload of large backups
+- Postgres addon whitelists pg_trgm and hstore extensions
+- Suppress boring update emails from patch releases
+- Setup bounce alerts for emails
+- Query admin's name in activation wizard
+- Admin emails are now delivered as no-reply
+- Fix crash when user attempts to set a duplicate email
+- Improved mongodb crash recovery
+
+[0.12.1]
+- Fix crash when backing up apps
+
+[0.12.2]
+- Improved error handling for addons
+
+[0.12.3]
+- LDAP: Do not set sn attribute when user has no surname
+
+[0.12.4]
+- Install app only after platform is ready
+
+[0.12.5]
+- Get alerts for app task failures
+- Fix update issue when one or more apps are in failed state
+
+[0.12.6]
+- Allow setting an alternate external domain for apps
+
+[0.12.7]
+- Fix changing password
+
+[0.13.0]
+- Upgrade to ubuntu 16.04
+- Add event log
+
+[0.13.1]
+- Make activity log viewable to admins
+- Fix geoip lookup
+
+[0.13.2]
+- Fix crash in app auto updater
+- Fix crash with empty timezone
+
+[0.13.3]
+- Enable auth in email addon
+- Add search for activity log
+- Add tutorial for first time users
+
+[0.13.4]
+- Fix mail addon restart issue
+
+[0.14.0]
+- You have mail :-)
+
+[0.14.1]
+- 2-character usernames are now allowed
+- Make cloudron CLI push/pull more robust
+
+[0.14.2]
+- Update mail addon
+
+[0.15.0]
+- [REST API](https://cloudron.io/references/api.html) is now in public beta
+- Enable Developer mode by default for new Cloudrons
+- Reverse proxy fixes for apps exposing a WebDav server
+- Allow admins to optionally set the username and displayName on user creation
+- Fix app autoupdate logic to detect if one or more in-use port bindings was removed
+
+[0.15.1]
+- Fix mail connectivity from IPv6 clients
+- Add API token management UI
+- Improved UI to enter email aliases
+
+[0.15.2]
+- Allow restoring apps from any previous backup
+

baseimage/createImage

@@ -137,8 +137,8 @@ while true; do
     sleep 30
 done
 
-echo "Copying INFRA_VERSION"
-$scp22 "${SCRIPT_DIR}/../setup/INFRA_VERSION" root@${server_ip}:.
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" root@${server_ip}:.
 
 echo "Copying box source"
 cd "${SOURCE_DIR}"

baseimage/digitalocean.sh

@@ -10,7 +10,7 @@ if [[ -z "${JSON}" ]]; then
     exit 1
 fi
 
-readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
 
 function debug() {
     echo "$@" >&2
@@ -30,7 +30,7 @@ function create_droplet() {
     local box_name="$2"
 
     local image_region="sfo1"
-    local ubuntu_image_slug="ubuntu-15-10-x64"
+    local ubuntu_image_slug="ubuntu-16-04-x64"
     local box_size="512mb"
 
     local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
@@ -109,13 +109,23 @@ function get_image_id() {
     local snapshot_name="$1"
     local image_id=""
 
-    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
-
-    if [[ -n "${image_id}" ]]; then
-        echo "${image_id}"
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=100"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
     fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
 }
 
 function snapshot_droplet() {
@@ -128,16 +138,26 @@ function snapshot_droplet() {
     debug -n "Waiting for snapshot to complete"
 
     while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
         if [[ "${event_status}" == "completed" ]]; then
             break
         fi
         debug -n "."
         sleep 10
     done
-    debug ""
+    debug "! done"
 
-    get_image_id "${snapshot_name}"
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
 }
 
 function destroy_droplet() {

baseimage/initializeBaseUbuntuImage.sh

@@ -17,13 +17,7 @@ function die {
     exit 1
 }
 
-[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
-
-if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
-    source "${SOURCE_DIR}/INFRA_VERSION"
-else
-    echo "No INFRA_VERSION found, skip pulling docker images"
-fi
+[[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"
 
 if [ ${SELFHOSTED} == 0 ]; then
     echo "!! Initializing Ubuntu image for CaaS"
@@ -49,7 +43,7 @@ export DEBIAN_FRONTEND=noninteractive
 
 echo "=== Upgrade ==="
 apt-get update
-apt-get upgrade -y
+apt-get dist-upgrade -y
 apt-get install -y curl
 
 # Setup firewall before everything. docker creates it's own chain and the -X below will remove it
@@ -67,9 +61,9 @@ iptables -P OUTPUT ACCEPT
 # allow ssh, http, https, ping, dns
 iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
 if [ ${SELFHOSTED} == 0 ]; then
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
 else
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,22,443,587,993,4190 -j ACCEPT
 fi
 iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
@@ -94,7 +88,8 @@ apt-get -y install btrfs-tools
 
 echo "==== Install docker ===="
 # install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
-curl https://get.docker.com/builds/Linux/x86_64/docker-1.9.1 > /usr/bin/docker
+curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
+apt-get -y install aufs-tools
 chmod +x /usr/bin/docker
 groupadd docker
 cat > /etc/systemd/system/docker.socket <<EOF
@@ -129,10 +124,10 @@ WantedBy=multi-user.target
 EOF
 
 echo "=== Setup btrfs data ==="
-fallocate -l "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
+truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
 mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
-echo "${USER_DATA_FILE} ${USER_DATA_DIR} btrfs loop,nosuid 0 0" >> /etc/fstab
-mkdir -p "${USER_DATA_DIR}" && mount "${USER_DATA_FILE}"
+mkdir -p "${USER_DATA_DIR}"
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
 
 systemctl daemon-reload
 systemctl enable docker
@@ -155,34 +150,26 @@ update-grub
 # now add the user to the docker group
 usermod "${USER}" -a -G docker
 
-if [ -z $(echo "${INFRA_VERSION}") ]; then
-    echo "Skip pulling base docker images"
-else
-    echo "=== Pulling base docker images ==="
-    docker pull "${BASE_IMAGE}"
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
-    echo "=== Pulling mysql addon image ==="
-    docker pull "${MYSQL_IMAGE}"
+echo "==== Downloading docker images ===="
+images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage, Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-    echo "=== Pulling postgresql addon image ==="
-    docker pull "${POSTGRESQL_IMAGE}"
-
-    echo "=== Pulling redis addon image ==="
-    docker pull "${REDIS_IMAGE}"
-
-    echo "=== Pulling mongodb addon image ==="
-    docker pull "${MONGODB_IMAGE}"
-
-    echo "=== Pulling graphite docker images ==="
-    docker pull "${GRAPHITE_IMAGE}"
-
-    echo "=== Pulling mail relay ==="
-    docker pull "${MAIL_IMAGE}"
-fi
+echo "Pulling images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+done
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
-[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
+[[ "$(nginx -v 2>&1)" == *"nginx/1.10."* ]] || die "Expecting nginx version to be 1.10.x"
 
 echo "==== Install build-essential ===="
 apt-get -y install build-essential rcconf
@@ -190,11 +177,11 @@ apt-get -y install build-essential rcconf
 echo "==== Install mysql ===="
 debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
 debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-apt-get -y install mysql-server
-[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
+apt-get -y install mysql-server-5.7
+[[ "$(mysqld --version 2>&1)" == *"5.7."* ]] || die "Expecting mysql version to be 5.7.x"
 
-echo "==== Install pwgen ===="
-apt-get -y install pwgen
+echo "==== Install pwgen and swaks awscli ===="
+apt-get -y install pwgen swaks awscli
 
 echo "==== Install collectd ==="
 if ! apt-get install -y collectd collectd-utils; then
@@ -209,15 +196,6 @@ echo "==== Install logrotate ==="
 apt-get install -y cron logrotate
 systemctl enable cron
 
-echo "==== Install nodejs ===="
-# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
-mkdir -p /usr/local/node-4.1.1
-curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
-ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
-ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
 echo "=== Rebuilding npm packages ==="
 cd "${INSTALLER_SOURCE_DIR}" && npm install --production
 chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
@@ -268,7 +246,7 @@ echo "==== Install box-setup systemd script ===="
 cat > /etc/systemd/system/box-setup.service <<EOF
 [Unit]
 Description=Box Setup
-Before=docker.service umount.target collectd.service
+Before=docker.service collectd.service mysql.service
 After=do-resize.service
 
 [Service]

crashnotifier.js

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();

crashnotifierservice.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendFailureLogs(processName, { unit: processName });
+}
+
+main();

gulpfile.js

@@ -10,7 +10,7 @@ var ejs = require('gulp-ejs'),
     serve = require('gulp-serve'),
     sass = require('gulp-sass'),
     sourcemaps = require('gulp-sourcemaps'),
-    minifyCSS = require('gulp-minify-css'),
+    cssnano = require('gulp-cssnano'),
     autoprefixer = require('gulp-autoprefixer'),
     argv = require('yargs').argv;
 
@@ -22,6 +22,7 @@ gulp.task('3rdparty', function () {
         'webadmin/src/3rdparty/**/*.otf',
         'webadmin/src/3rdparty/**/*.eot',
         'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.gif',
         'webadmin/src/3rdparty/**/*.ttf',
         'webadmin/src/3rdparty/**/*.woff',
         'webadmin/src/3rdparty/**/*.woff2'
@@ -119,7 +120,7 @@ gulp.task('css', function () {
         .pipe(sourcemaps.init())
         .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
         .pipe(autoprefixer())
-        .pipe(minifyCSS())
+        .pipe(cssnano())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist'))
         .pipe(gulp.dest('setup/splash/website'));

installer.sh.ejs

@@ -20,10 +20,15 @@ readonly provider="${5}"
 readonly revision="${6}"
 
 # environment specific urls
-readonly api_server_origin="https://api.dev.cloudron.io"
-readonly web_server_origin="https://dev.cloudron.io"
-readonly release_bucket_url="https://s3.amazonaws.com/dev-cloudron-releases"
-readonly versions_url="https://s3.amazonaws.com/dev-cloudron-releases/versions.json"
+<% if (env === 'prod') { %>
+readonly api_server_origin="https://api.cloudron.io"
+readonly web_server_origin="https://cloudron.io"
+<% } else { %>
+readonly api_server_origin="https://api.<%= env %>.cloudron.io"
+readonly web_server_origin="https://<%= env %>.cloudron.io"
+<% } %>
+readonly release_bucket_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases"
+readonly versions_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases/versions.json"
 readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
 
 # runtime consts
@@ -132,7 +137,7 @@ cat > /root/provision.json <<EOF
             "secretAccessKey": "${aws_access_key_secret}"
         },
         "tlsConfig": {
-            "provider": "letsencrypt-dev"
+            "provider": "letsencrypt-<%= env %>"
         }
     }
 }

installer/src/scripts/installer.sh

@@ -7,7 +7,7 @@ readonly DATA_DIR=/home/yellowtent/data
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly json="${script_dir}/../../node_modules/.bin/json"
-readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 180"
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
 
 readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
 

installer/systemd/box-setup.sh

@@ -4,7 +4,6 @@ set -eu -o pipefail
 
 readonly USER_HOME="/home/yellowtent"
 readonly APPS_SWAP_FILE="/apps.swap"
-readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
@@ -19,12 +18,10 @@ fi
 
 # all sizes are in mb
 readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
-readonly swap_size="${physical_memory}"
+readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
 readonly app_count=$((${physical_memory} / 200)) # estimated app count
 readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
 readonly disk_size=$((disk_size_gb * 1024))
-readonly backup_swap_size=1024
-# readonly system_size=5120 # 5 gigs for system libs, installer, box code and tmp
 readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
 readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
 
@@ -33,8 +30,7 @@ echo "Physical memory: ${physical_memory}"
 echo "Estimated app count: ${app_count}"
 echo "Disk size: ${disk_size}"
 
-# Allocate two sets of swap files - one for general app usage and another for backup
-# The backup swap is setup for swap on the fly by the backup scripts
+# Allocate swap for general app usage
 if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
     echo "Creating Apps swap file of size ${swap_size}M"
     fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
@@ -46,20 +42,13 @@ else
     echo "Apps Swap file already exists"
 fi
 
-if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then
-    echo "Creating Backup swap file of size ${backup_swap_size}M"
-    fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}"
-    chmod 600 "${BACKUP_SWAP_FILE}"
-    mkswap "${BACKUP_SWAP_FILE}"
-else
-    echo "Backups Swap file already exists"
-fi
-
 echo "Resizing data volume"
-home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
+home_data_size=$((disk_size - system_size - swap_size - ext4_reserved))
 echo "Resizing up btrfs user data to size ${home_data_size}M"
-umount "${USER_DATA_DIR}"
-fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
-mount "${USER_DATA_FILE}"
+umount "${USER_DATA_DIR}" || true
+# Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data
+# fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
+truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
 btrfs filesystem resize max "${USER_DATA_DIR}"
 

migrations/20160205135326-apps-add-memoryLimit.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN memoryLimit BIGINT DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN memoryLimit', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208031241-groups-add-table.js

@@ -0,0 +1,21 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE groups(" +
+                "id VARCHAR(128) NOT NULL UNIQUE," +
+                "name VARCHAR(128) NOT NULL UNIQUE," +
+                "PRIMARY KEY(id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208100000-groupMembers-add-table.js

@@ -0,0 +1,22 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = "CREATE TABLE IF NOT EXISTS groupMembers(" +
+    			"groupId VARCHAR(128) NOT NULL," +
+    			"userId VARCHAR(128) NOT NULL," +
+    			"FOREIGN KEY(groupId) REFERENCES groups(id)," +
+    			"FOREIGN KEY(userId) REFERENCES users(id));";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groupMembers', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208164735-groups-add-admin.js

@@ -0,0 +1,30 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var async = require('async');
+
+var ADMIN_GROUP_ID = 'admin'; // see groups.js
+
+exports.up = function(db, callback) {
+	async.series([
+		db.runSql.bind(db, 'START TRANSACTION;'),
+		db.runSql.bind(db, 'INSERT INTO groups (id, name) VALUES (?, ?)', [ ADMIN_GROUP_ID, 'admin' ]),
+		function migrateAdminFlag(done) {
+			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
+				if (error) return done(error);
+
+				console.dir(results);
+
+				async.eachSeries(results, function (r, next) {
+					db.runSql('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ ADMIN_GROUP_ID, r.id ], next);
+				}, done);
+			});
+		},
+		db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+		db.runSql.bind(db, 'COMMIT')
+	], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20160307172229-backups-add-table.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE backups(" +
+            "filename VARCHAR(128) NOT NULL," +
+            "creationTime TIMESTAMP," +
+            "version VARCHAR(128) NOT NULL," +
+            "type VARCHAR(16) NOT NULL," +
+            "dependsOn VARCHAR(4096)," +
+            "state VARCHAR(16) NOT NULL," +
+            "PRIMARY KEY (filename))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE backups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160330215005-backups-add-configJson.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20160404052453-backups-drop-configJson.js

@@ -0,0 +1,17 @@
+var dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20160404191651-backups-rename-filename-to-id.js

@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE filename id VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE id filename VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160405083451-users-drop-username-null.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160419070047-apps-add-altDomain.js

@@ -0,0 +1,17 @@
+'use strict';
+
+var dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN altDomain VARCHAR(256)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN altDomain', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160430062446-eventlog-add-table.js

@@ -0,0 +1,24 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE eventlog(" +
+            "id VARCHAR(128) NOT NULL," +
+            "source JSON," +
+            "creationTime TIMESTAMP," +
+            "action VARCHAR(128) NOT NULL," +
+            "data JSON," +
+            "PRIMARY KEY (id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE eventlog', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160506111925-users-add-showTutorial.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN showTutorial BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN showTutorial', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160527010012-mailboxes-add-table.js

@@ -0,0 +1,24 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160528090040-mailboxes-import-existing-users.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20160614021819-apps-drop-lastBackupConfigJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -11,7 +11,7 @@
 
 CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(254) NOT NULL UNIQUE,
+    username VARCHAR(254) UNIQUE,
     email VARCHAR(254) NOT NULL UNIQUE,
     password VARCHAR(1024) NOT NULL,
     salt VARCHAR(512) NOT NULL,
@@ -19,14 +19,26 @@ CREATE TABLE IF NOT EXISTS users(
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     displayName VARCHAR(512) DEFAULT '',
+    showTutorial BOOLEAN DEFAULT 0,
     PRIMARY KEY(id));
 
+CREATE TABLE IF NOT EXISTS groups(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS groupMembers(
+    groupId VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    FOREIGN KEY(groupId) REFERENCES groups(id),
+    FOREIGN KEY(userId) REFERENCES users(id));
+
 CREATE TABLE IF NOT EXISTS tokens(
     accessToken VARCHAR(128) NOT NULL UNIQUE,
     identifier VARCHAR(128) NOT NULL,
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
@@ -50,14 +62,15 @@ CREATE TABLE IF NOT EXISTS apps(
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    memoryLimit BIGINT DEFAULT 0,
+    altDomain VARCHAR(256),
 
-    lastBackupId VARCHAR(128),
-    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupId VARCHAR(128), // tracks last valid backup, can be removed
 
-    oldConfigJson TEXT, // used to pass old config for apptask
+    oldConfigJson TEXT, // used to pass old config for apptask, can be removed when we use a queue
 
     PRIMARY KEY(id));
 
@@ -72,7 +85,7 @@ CREATE TABLE IF NOT EXISTS authcodes(
     authCode VARCHAR(128) NOT NULL UNIQUE,
     userId VARCHAR(128) NOT NULL,
     clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
@@ -86,3 +99,32 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     value VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
+CREATE TABLE IF NOT EXISTS backups(
+    filename VARCHAR(128) NOT NULL,
+    creationTime TIMESTAMP,
+    version VARCHAR(128) NOT NULL, /* app version or box version */
+    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
+    dependsOn VARCHAR(4096), /* comma separate list of objects this backup depends on */
+    state VARCHAR(16) NOT NULL,
+
+    PRIMARY KEY (filename));
+
+CREATE TABLE IF NOT EXISTS eventlog(
+    id VARCHAR(128) NOT NULL,
+    action VARCHAR(128) NOT NULL,
+    source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+
+    PRIMARY KEY (id));
+
+/* Future fields:
+   * accessRestriction - to determine who can access it. So this has foreign keys
+   * quota - per mailbox quota
+*/
+CREATE TABLE IF NOT EXISTS mailboxes(
+    name VARCHAR(128) NOT NULL,
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
+    creationTime TIMESTAMP,
+
+    PRIMARY KEY (id));

package.json

@@ -14,13 +14,11 @@
   ],
   "dependencies": {
     "async": "^1.2.1",
-    "attempt": "^1.0.1",
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "bytes": "^2.1.0",
-    "cloudron-manifestformat": "^2.2.0",
+    "cloudron-manifestformat": "^2.4.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
+    "connect-lastmile": "^0.1.0",
     "connect-timeout": "^1.5.0",
     "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
@@ -28,17 +26,17 @@
     "csurf": "^1.6.6",
     "db-migrate": "^0.9.2",
     "debug": "^2.2.0",
-    "dockerode": "^2.2.2",
+    "dockerode": "^2.2.10",
     "ejs": "^2.2.4",
-    "ejs-cli": "^1.0.1",
+    "ejs-cli": "^1.2.0",
     "express": "^4.12.4",
     "express-session": "^1.11.3",
     "hat": "0.0.3",
+    "ini": "^1.3.4",
     "json": "^9.0.3",
     "ldapjs": "^0.7.1",
-    "memorystream": "^0.3.0",
     "mime": "^1.3.4",
-    "morgan": "^1.6.0",
+    "morgan": "^1.7.0",
     "multiparty": "^4.1.2",
     "mysql": "^2.7.0",
     "native-dns": "^0.7.0",
@@ -48,6 +46,7 @@
     "nodemailer-smtp-transport": "^1.0.3",
     "oauth2orize": "^1.0.1",
     "once": "^1.3.2",
+    "parse-links": "^0.1.0",
     "passport": "^0.2.2",
     "passport-http": "^0.2.2",
     "passport-http-bearer": "^1.0.1",
@@ -55,18 +54,18 @@
     "passport-oauth2-client-password": "^0.1.2",
     "password-generator": "^2.0.2",
     "proxy-middleware": "^0.13.0",
-    "safetydance": "^0.1.0",
+    "safetydance": "^0.1.1",
     "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
-    "superagent": "^1.5.0",
+    "superagent": "^1.8.3",
     "supererror": "^0.7.1",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "tldjs": "^1.6.2",
     "underscore": "^1.7.0",
-    "ursa": "^0.9.1",
+    "ursa": "^0.9.3",
     "valid-url": "^1.0.9",
-    "validator": "^4.4.0",
-    "x509": "^0.2.2"
+    "validator": "^4.9.0",
+    "x509": "^0.2.4"
   },
   "devDependencies": {
     "apidoc": "*",
@@ -77,8 +76,8 @@
     "gulp": "^3.8.11",
     "gulp-autoprefixer": "^2.3.0",
     "gulp-concat": "^2.4.3",
+    "gulp-cssnano": "^2.1.0",
     "gulp-ejs": "^1.0.0",
-    "gulp-minify-css": "^1.1.3",
     "gulp-sass": "^2.0.1",
     "gulp-serve": "^1.0.0",
     "gulp-sourcemaps": "^1.5.2",
@@ -89,7 +88,6 @@
     "mocha": "*",
     "nock": "^3.4.0",
     "node-sass": "^3.0.0-alpha.0",
-    "redis": "^2.4.2",
     "request": "^2.65.0",
     "sinon": "^1.12.2",
     "yargs": "^3.15.0"

setup/INFRA_VERSION

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=21
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.8.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.8.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.9.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -21,6 +21,7 @@ arg_backup_config=""
 arg_dns_config=""
 arg_update_config=""
 arg_provider=""
+arg_app_bundle=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -37,6 +38,9 @@ while true; do
         $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
 EOF
         # read possibly empty parameters here
+        arg_app_bundle=$(echo "$2" | $json appBundle)
+        [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"
+
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
         arg_token=$(echo "$2" | $json token)

setup/container/mysql.cnf

@@ -4,4 +4,4 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connection=50
+max_connections=50

setup/container/sudoers

@@ -25,12 +25,12 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
 
-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 
 Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
 
+Defaults!/home/yellowtent/box/src/scripts/setup_infra.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setup_infra.sh
+

setup/container/systemd/cloudron.target

@@ -1,5 +1,5 @@
 [Unit]
-Description=Cloudron Smart Cloud
+Description=Cloudron Smartserver
 Documentation=https://cloudron.io/documentation.html
 StopWhenUnneeded=true
 Requires=box.service

setup/container/systemd/crashnotifier@.service

@@ -7,7 +7,7 @@ StopWhenUnneeded=false
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
-ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+ExecStart="/home/yellowtent/box/crashnotifierservice.js" %I
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 KillMode=process
 User=yellowtent

setup/splashpage.sh

@@ -9,8 +9,6 @@ readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
 readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
-source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
-
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
@@ -24,13 +22,16 @@ rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+readonly current_infra=$(node -e "console.log(require('${script_dir}/../src/infra_version.js').version);")
+existing_infra="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && existing_infra=$(node -e "console.log(JSON.parse(require('fs').readFileSync('${DATA_DIR}/INFRA_VERSION', 'utf8')).version);")
+if [[ "${arg_retire}" == "true" || "${existing_infra}" != "${current_infra}" ]]; then
+    echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire} existing: ${existing_infra} current: ${current_infra}"
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
         -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
+    echo "Show progress bar only on admin domain for normal update"
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
         -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi

setup/start.sh

@@ -39,7 +39,7 @@ set_progress "10" "Ensuring directories"
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
 mkdir -p "${DATA_DIR}/box/appicons"
 mkdir -p "${DATA_DIR}/box/certs"
-mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/box/mail/dkim/${arg_fqdn}"
 mkdir -p "${DATA_DIR}/box/acme" # acme keys
 mkdir -p "${DATA_DIR}/graphite"
 
@@ -120,11 +120,9 @@ fi
 
 set_progress "33" "Changing ownership"
 chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"
 
-set_progress "40" "Setting up infra"
-${script_dir}/start/setup_infra.sh "${arg_fqdn}"
-
 set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
@@ -138,7 +136,6 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "admin@${arg_fqdn}",
     "provider": "${arg_provider}",
     "database": {
         "hostname": "localhost",
@@ -146,7 +143,8 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "password": "${mysql_root_password}",
         "port": 3306,
         "name": "box"
-    }
+    },
+    "appBundle": ${arg_app_bundle}
 }
 CONF_END
 
@@ -190,18 +188,20 @@ if [[ ! -z "${arg_tls_config}" ]]; then
         -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
 fi
 
-# Add webadmin oauth client
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add webadmin api cient"
+readonly ADMIN_SCOPES="cloudron,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"Settings\", \"built-in\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add SDK api client"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-sdk\", \"SDK\", \"built-in\", \"secret-sdk\", \"${admin_origin}\", \"*,roleSdk\")" box
+
+echo "Add cli api client"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-cli\", \"Cloudron Tool\", \"built-in\", \"secret-cli\", \"${admin_origin}\", \"*,roleSdk\")" box
 
 set_progress "80" "Starting Cloudron"
 systemctl start cloudron.target

setup/start/nginx/appconfig.ejs

@@ -18,9 +18,10 @@ server {
     # https://bettercrypto.org/static/applied-crypto-hardening.pdf
     # https://mozilla.github.io/server-side-tls/ssl-config-generator/
     # https://cipherli.st/
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
     ssl_prefer_server_ciphers on;
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
     add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
 
     proxy_http_version 1.1;
@@ -43,6 +44,18 @@ server {
         return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
     }
 
+<% if ( endpoint === 'app' ) { %>
+    # For some reason putting this webdav block inside location does not work
+    # http://serverfault.com/questions/121766/webdav-rename-fails-on-an-apache-mod-dav-install-behind-nginx
+    if ($request_method ~ ^(COPY|MOVE)$) {
+        set $destination $http_destination;
+    }
+    if ($destination ~* ^https(.+)$) {
+        set $destination http$1;
+    }
+    proxy_set_header Destination $destination;
+<% } %>
+
     location / {
         # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
         proxy_buffer_size       128k;
@@ -58,6 +71,7 @@ server {
             client_max_body_size 1m;
         }
 
+        # the read timeout is between successive reads and not the whole connection
         location ~ ^/api/v1/apps/.*/exec$ {
             proxy_pass   http://127.0.0.1:3000;
             proxy_read_timeout 30m;

setup/start/nginx/nginx.conf

@@ -24,7 +24,14 @@ http {
 
     sendfile        on;
 
-    keepalive_timeout  65;
+    # timeout for client to finish sending headers
+    client_header_timeout 30s;
+
+    # timeout for reading client request body (successive read timeout and not whole body!)
+    client_body_timeout 60s;
+
+    # keep-alive connections timeout in 65s. this is because many browsers timeout in 60 seconds
+    keepalive_timeout  65s;
 
     # HTTP server
     server {
@@ -50,22 +57,15 @@ http {
         }
     }
 
-    # We have to enable https for nginx to read in the vhost in http request
-    # and send a 404. This is a side-effect of using wildcard DNS
+    # This server handles the naked domain for custom domains.
+    # It can also be used for wildcard subdomain 404. This feature is not used by the Cloudron itself
+    # because box always sets up DNS records for app subdomains.
     server {
         listen 443 default_server;
         ssl on;
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # Disable check to allow unlimited body sizes
-        client_max_body_size 0;
-
         error_page 404 = @fallback;
         location @fallback {
             internal;
@@ -79,6 +79,7 @@ http {
             rewrite ^/$ /nakeddomain.html break;
         }
 
+        # required for /api/v1/cloudron/avatar
         location /api/ {
             proxy_pass http://127.0.0.1:3000;
             client_max_body_size 1m;

setup/start/setup_infra.sh

@@ -1,129 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-arg_fqdn="$1"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"

src/addons.js

@@ -19,30 +19,34 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    child_process = require('child_process'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
-    DatabaseError = require('./databaseerror.js'),
+    ClientsError = clients.ClientsError,
     debug = require('debug')('box:addons'),
-    docker = require('./docker.js').connection,
+    docker = require('./docker.js'),
+    dockerConnection = docker.connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
-    MemoryStream = require('memorystream'),
+    infra = require('./infra_version.js'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
-    spawn = child_process.spawn,
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
 
 var NOOP = function (app, options, callback) { return callback(); };
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    email: {
+        setup: setupEmail,
+        teardown: teardownEmail,
+        backup: NOOP,
+        restore: setupEmail
+    },
     ldap: {
         setup: setupLdap,
         teardown: teardownLdap,
@@ -79,6 +83,12 @@ var KNOWN_ADDONS = {
         backup: backupPostgreSql,
         restore: restorePostgreSql
     },
+    recvmail: {
+        setup: setupRecvMail,
+        teardown: teardownRecvMail,
+        backup: NOOP,
+        restore: setupRecvMail
+    },
     redis: {
         setup: setupRedis,
         teardown: teardownRedis,
@@ -207,17 +217,23 @@ function getLinksSync(app, addons) {
 
     if (!addons) return links;
 
+    var addMail = false;
+
     for (var addon in addons) {
         switch (addon) {
         case 'mysql': links.push('mysql:mysql'); break;
         case 'postgresql': links.push('postgresql:postgresql'); break;
-        case 'sendmail': links.push('mail:mail'); break;
+        case 'sendmail': addMail = true; break;
+        case 'recvmail': addMail = true; break;
+        case 'email': addMail = true; break;
         case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
         case 'mongodb': links.push('mongodb:mongodb'); break;
         default: break;
         }
     }
 
+    if (addMail) links.push('mail:mail');
+
     return links;
 }
 
@@ -267,22 +283,18 @@ function setupOauth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
+    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'OAUTH_CLIENT_ID=' + id,
-                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_CLIENT_ID=' + result.id,
+                'OAUTH_CLIENT_SECRET=' + result.clientSecret,
                 'OAUTH_ORIGIN=' + config.adminOrigin()
             ];
 
@@ -300,8 +312,8 @@ function teardownOauth(app, options, callback) {
 
     debugApp(app, 'teardownOauth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
@@ -313,15 +325,12 @@ function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
     var scope = 'profile';
 
-    debugApp(app, 'setupSimpleAuth: id:%s', id);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+        clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
@@ -329,7 +338,7 @@ function setupSimpleAuth(app, options, callback) {
                 'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
                 'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
                 'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_CLIENT_ID=' + id
+                'SIMPLE_AUTH_CLIENT_ID=' + result.id
             ];
 
             debugApp(app, 'Setting simple auth addon config to %j', env);
@@ -346,13 +355,44 @@ function teardownSimpleAuth(app, options, callback) {
 
     debugApp(app, 'teardownSimpleAuth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
     });
 }
 
+function setupEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2525',
+        'MAIL_IMAP_SERVER=mail',
+        'MAIL_IMAP_PORT=9993',
+        'MAIL_SIEVE_SERVER=mail',
+        'MAIL_SIEVE_PORT=4190',
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up Email');
+
+    appdb.setAddonConfig(app.id, 'email', env, callback);
+}
+
+function teardownEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down Email');
+
+    appdb.unsetAddonConfig(app.id, 'email', callback);
+}
+
 function setupLdap(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
@@ -388,18 +428,17 @@ function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var username = app.location ? app.location + '-app' : 'no-reply'; // use no-reply for bare domains
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
 
-    var env = [
-        'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + username,
-        'MAIL_DOMAIN=' + config.fqdn()
-    ];
+    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];
 
-    debugApp(app, 'Setting up sendmail');
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
 
-    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting sendmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    });
 }
 
 function teardownSendMail(app, options, callback) {
@@ -409,7 +448,55 @@ function teardownSendMail(app, options, callback) {
 
     debugApp(app, 'Tearing down sendmail');
 
-    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
+
+    debugApp(app, 'Tearing down sendmail');
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    });
+}
+
+function setupRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up recvmail');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
+
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting recvmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'recvmail', env, callback);
+    });
+}
+
+function teardownRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
+
+    debugApp(app, 'Tearing down recvmail');
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
+    });
 }
 
 function setupMySql(app, options, callback) {
@@ -419,31 +506,14 @@ function setupMySql(app, options, callback) {
 
     debugApp(app, 'Setting up mysql');
 
-    var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'add', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mysql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mysql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mysql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mysql', env, callback);
     });
 }
 
@@ -452,24 +522,14 @@ function teardownMySql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];
 
     debugApp(app, 'Tearing down mysql');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mysql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mysql', callback);
     });
 }
 
@@ -481,15 +541,9 @@ function backupMySql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mysql', cmd, { stdout: output }, callback);
 }
 
 function restoreMySql(app, options, callback) {
@@ -503,17 +557,8 @@ function restoreMySql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMySql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ];
+        docker.execContainer('mysql', cmd, { stdin: input }, callback);
     });
 }
 
@@ -524,31 +569,14 @@ function setupPostgreSql(app, options, callback) {
 
     debugApp(app, 'Setting up postgresql');
 
-    var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting postgresql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'postgresql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting postgresql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'postgresql', env, callback);
     });
 }
 
@@ -557,24 +585,14 @@ function teardownPostgreSql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
 
     debugApp(app, 'Tearing down postgresql');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'postgresql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'postgresql', callback);
     });
 }
 
@@ -586,19 +604,13 @@ function backupPostgreSql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/postgresql/service.sh', 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('postgresql', cmd, { stdout: output }, callback);
 }
 
 function restorePostgreSql(app, options, callback) {
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    callback = once(callback);
 
     setupPostgreSql(app, options, function (error) {
         if (error) return callback(error);
@@ -608,17 +620,9 @@ function restorePostgreSql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
-        });
+        var cmd = [ '/addons/postgresql/service.sh', 'restore', app.id ];
 
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        docker.execContainer('postgresql', cmd, { stdin: input }, callback);
     });
 }
 
@@ -629,31 +633,14 @@ function setupMongoDb(app, options, callback) {
 
     debugApp(app, 'Setting up mongodb');
 
-    var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mongodb addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mongodb addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mongodb', env, callback);
     });
 }
 
@@ -662,24 +649,14 @@ function teardownMongoDb(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
 
     debugApp(app, 'Tearing down mongodb');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mongodb', callback);
     });
 }
 
@@ -691,15 +668,9 @@ function backupMongoDb(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mongodb/service.sh', 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mongodb', cmd, { stdout: output }, callback);
 }
 
 function restoreMongoDb(app, options, callback) {
@@ -713,26 +684,16 @@ function restoreMongoDb(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mongodb/service.sh', 'restore', app.id ];
+        docker.execContainer('mongodb', cmd, { stdin: input }, callback);
     });
 }
 
-
 function forwardRedisPort(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    docker.getContainer('redis-' + appId).inspect(function (error, data) {
+    dockerConnection.getContainer('redis-' + appId).inspect(function (error, data) {
         if (error) return callback(new Error('Unable to inspect container:' + error));
 
         var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
@@ -780,7 +741,7 @@ function setupRedis(app, options, callback) {
         name: 'redis-' + app.id,
         Hostname: 'redis-' + app.location,
         Tty: true,
-        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
+        Image: infra.images.redis.tag,
         Cmd: null,
         Volumes: {
             '/tmp': {},
@@ -812,9 +773,9 @@ function setupRedis(app, options, callback) {
         'REDIS_PORT=6379'
     ];
 
-    var redisContainer = docker.getContainer(createOptions.name);
+    var redisContainer = dockerConnection.getContainer(createOptions.name);
     stopAndRemoveRedis(redisContainer, function () {
-        docker.createContainer(createOptions, function (error) {
+        dockerConnection.createContainer(createOptions, function (error) {
             if (error && error.statusCode !== 409) return callback(error); // if not already created
 
             redisContainer.start(function (error) {
@@ -835,7 +796,7 @@ function teardownRedis(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-   var container = docker.getContainer('redis-' + app.id);
+   var container = dockerConnection.getContainer('redis-' + app.id);
 
    var removeOptions = {
        force: true, // kill container if it's running
@@ -858,15 +819,7 @@ function teardownRedis(app, options, callback) {
 function backupRedis(app, options, callback) {
     debugApp(app, 'Backing up redis');
 
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    var cmd = [ '/addons/redis/service.sh', 'backup' ]; // the redis dir is volume mounted
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
-    });
-
-    cp.stdout.pipe(process.stdout);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('redis-' + app.id, cmd, { }, callback);
 }

src/appdb.js

@@ -4,7 +4,6 @@
 
 exports = module.exports = {
     get: get,
-    getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
     getByContainerId: getByContainerId,
     add: add,
@@ -59,7 +58,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -70,10 +69,6 @@ function postProcess(result) {
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
-    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
-    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
-    delete result.lastBackupConfigJson;
-
     assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
     result.oldConfig = safe.JSON.parse(result.oldConfigJson);
     delete result.oldConfigJson;
@@ -92,8 +87,6 @@ function postProcess(result) {
         result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
     }
 
-    result.oauthProxy = !!result.oauthProxy;
-
     assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
     result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
     if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
@@ -116,22 +109,6 @@ function get(id, callback) {
     });
 }
 
-function getBySubdomain(subdomain, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
-    });
-}
-
 function getByHttpPort(httpPort, callback) {
     assert.strictEqual(typeof httpPort, 'number');
     assert.strictEqual(typeof callback, 'function');
@@ -179,7 +156,7 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, altDomain, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
@@ -187,7 +164,8 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert.strictEqual(typeof memoryLimit, 'number');
+    assert(altDomain === null || typeof altDomain === 'string');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
@@ -197,8 +175,8 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit, altDomain ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -283,6 +261,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     assert.strictEqual(typeof constraints, 'string');
     assert.strictEqual(typeof callback, 'function');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
 
     var queries = [ ];
 
@@ -301,9 +280,6 @@ function updateWithConstraints(id, app, constraints, callback) {
         if (p === 'manifest') {
             fields.push('manifestJson = ?');
             values.push(JSON.stringify(app[p]));
-        } else if (p === 'lastBackupConfig') {
-            fields.push('lastBackupConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
         } else if (p === 'oldConfig') {
             fields.push('oldConfigJson = ?');
             values.push(JSON.stringify(app[p]));

src/apphealthmonitor.js

@@ -3,6 +3,7 @@
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
+    config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apphealthmonitor'),
     docker = require('./docker.js').connection,
@@ -16,7 +17,7 @@ exports = module.exports = {
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
 var gHealthInfo = { }; // { time, emailSent }
 var gRunTimeout = null;
 var gDockerEventStream = null;
@@ -24,8 +25,11 @@ var gDockerEventStream = null;
 function debugApp(app) {
     assert(!app || typeof app === 'object');
 
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
+    var manifestAppId = app ? app.manifest.id : '';
+    var id = app ? app.id : '';
+
+    debug(prefix + ' ' + manifestAppId + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + id);
 }
 
 function setHealth(app, health, callback) {
@@ -89,6 +93,7 @@ function checkAppHealth(app, callback) {
         var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
         superagent
             .get(healthCheckUrl)
+            .set('Host', config.appFqdn(app.location)) // required for some apache configs with rewrite rules
             .redirects(0)
             .timeout(HEALTHCHECK_INTERVAL)
             .end(function (error, res) {
@@ -114,7 +119,7 @@ function processApps(callback) {
 
             var alive = apps
                 .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
-                .map(function (a) { return a.location; }).join(', ');
+                .map(function (a) { return (a.location || 'naked_domain') + '|' + a.manifest.id; }).join(', ');
 
             debug('apps alive: [%s]', alive);
 
@@ -157,7 +162,7 @@ function processDockerEvents() {
                 debug('OOM Context: %s', context);
 
                 // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+                if (error || app.appStoreId !== '') mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
             });
         });
 

src/apptask.js

@@ -1,7 +1,5 @@
 #!/usr/bin/env node
 
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -19,7 +17,8 @@ exports = module.exports = {
     _verifyManifest: verifyManifest,
     _registerSubdomain: registerSubdomain,
     _unregisterSubdomain: unregisterSubdomain,
-    _waitForDnsPropagation: waitForDnsPropagation
+    _waitForDnsPropagation: waitForDnsPropagation,
+    _waitForAltDomainDnsPropagation: waitForAltDomainDnsPropagation
 };
 
 require('supererror')({ splatchError: true });
@@ -35,16 +34,16 @@ var addons = require('./addons.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
+    backups = require('./backups.js'),
     certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    ClientsError = clients.ClientsError,
     database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
     fs = require('fs'),
-    hat = require('hat'),
     manifestFormat = require('cloudron-manifestformat'),
     net = require('net'),
     nginx = require('./nginx.js'),
@@ -57,7 +56,7 @@ var addons = require('./addons.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
+    waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
 var COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
@@ -100,9 +99,7 @@ function configureNginx(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var vhost = config.appFqdn(app.location);
-
-    certificates.ensureCertificate(vhost, function (error, certFilePath, keyFilePath) {
+    certificates.ensureCertificate(app, function (error, certFilePath, keyFilePath) {
         if (error) return callback(error);
 
         nginx.configureApp(app, certFilePath, keyFilePath, callback);
@@ -162,22 +159,20 @@ function allocateOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!app.oauthProxy) return callback(null);
+    if (!nginx.requiresOAuthProxy(app)) return callback(null);
 
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
 }
 
 function removeOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) {
             debugApp(app, 'Error removing OAuth client id', error);
             return callback(error);
         }
@@ -231,17 +226,19 @@ function downloadIcon(app, callback) {
 
     var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
 
-    superagent
-        .get(iconUrl)
-        .buffer(true)
-        .end(function (error, res) {
-            if (error && !error.response) return callback(new Error('Network error downloading icon:' + error.message));
-            if (res.statusCode !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        superagent
+            .get(iconUrl)
+            .buffer(true)
+            .end(function (error, res) {
+                if (error && !error.response) return retryCallback(new Error('Network error downloading icon:' + error.message));
+                if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli
 
-            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return retryCallback(new Error('Error saving icon:' + safe.error.message));
 
-            callback(null);
-    });
+                retryCallback(null);
+        });
+    }, callback);
 }
 
 function registerSubdomain(app, callback) {
@@ -318,20 +315,23 @@ function waitForDnsPropagation(app, callback) {
         return callback(null);
     }
 
-    function retry(error) {
-        debugApp(app, 'waitForDnsPropagation: ', error);
-        setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
-    }
+    async.retry({ interval: 5000, times: 120 }, function checkStatus(retryCallback) {
+        subdomains.status(app.dnsRecordId, function (error, result) {
+            if (error) return retryCallback(new Error('Failed to get dns record status : ' + error.message));
 
-    subdomains.status(app.dnsRecordId, function (error, result) {
-        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
+            if (result !== 'done') return retryCallback(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
+            retryCallback(null, result);
+        });
+    }, callback);
+}
 
-        callback(null);
-    });
+function waitForAltDomainDnsPropagation(app, callback) {
+    if (!app.altDomain) return callback(null);
+
+    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', callback); // waits forever
 }
 
 // updates the app object and the database
@@ -410,9 +410,12 @@ function install(app, callback) {
 
         runApp.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
+
         updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
         configureNginx.bind(null, app),
 
@@ -436,7 +439,7 @@ function backup(app, callback) {
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        apps.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest),
 
         // done!
         function (callback) {
@@ -501,7 +504,7 @@ function restore(app, callback) {
         createVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons, backupId),
+        backups.restoreApp.bind(null, app, app.manifest.addons, backupId),
 
         updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
         createContainer.bind(null, app),
@@ -511,9 +514,12 @@ function restore(app, callback) {
 
         runApp.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
         configureNginx.bind(null, app),
 
@@ -573,6 +579,9 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
         configureNginx.bind(null, app),
 
@@ -598,19 +607,24 @@ function update(app, callback) {
     debugApp(app, 'Updating to %s', safe.query(app, 'manifest.version'));
 
     // app does not want these addons anymore
+    // FIXME: this does not handle option changes (like multipleDatabases)
     var unusedAddons = _.omit(app.oldConfig.manifest.addons, Object.keys(app.manifest.addons));
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '0, Verify manifest' }),
         verifyManifest.bind(null, app),
 
+        // download new image before app is stopped. this is so we can reduce downtime
+        // and also not remove the 'common' layers when the old image is deleted
+        updateApp.bind(null, app, { installationProgress: '15, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
         // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
         // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
-        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        updateApp.bind(null, app, { installationProgress: '25, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainers.bind(null, app),
-        addons.teardownAddons.bind(null, app, unusedAddons),
         function deleteImageIfChanged(done) {
              if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
 
@@ -622,16 +636,16 @@ function update(app, callback) {
             if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);
 
             async.series([
-                updateApp.bind(null, app, { installationProgress: '20, Backup app' }),
-                apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
+                backups.backupApp.bind(null, app, app.oldConfig.manifest)
             ], next);
         },
 
-        updateApp.bind(null, app, { installationProgress: '35, Downloading icon' }),
-        downloadIcon.bind(null, app),
+        // only delete unused addons after backup
+        addons.teardownAddons.bind(null, app, unusedAddons),
 
-        updateApp.bind(null, app, { installationProgress: '45, Downloading image' }),
-        docker.downloadImage.bind(null, app.manifest),
+        updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
+        downloadIcon.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '70, Updating addons' }),
         addons.setupAddons.bind(null, app, app.manifest.addons),
@@ -697,7 +711,13 @@ function uninstall(app, callback) {
 
         updateApp.bind(null, app, { installationProgress: '95, Remove app from database' }),
         appdb.del.bind(null, app.id)
-    ], callback);
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error uninstalling app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
 }
 
 function runApp(app, callback) {
@@ -760,7 +780,7 @@ function startTask(appId, callback) {
         case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
         case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
         case appdb.ISTATE_ERROR:
-            debugApp(app, 'Apptask launched with error states.');
+            debugApp(app, 'Internal error. apptask launched with error status.');
             return callback(null);
         default:
             debugApp(app, 'apptask launched with invalid command');

src/auth.js

@@ -10,8 +10,9 @@ exports = module.exports = {
 var assert = require('assert'),
     BasicStrategy = require('passport-http').BasicStrategy,
     BearerStrategy = require('passport-http-bearer').Strategy,
-    clientdb = require('./clientdb'),
+    clients = require('./clients'),
     ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    ClientsError = clients.ClientsError,
     DatabaseError = require('./databaseerror'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
@@ -19,7 +20,6 @@ var assert = require('assert'),
     passport = require('passport'),
     tokendb = require('./tokendb'),
     user = require('./user'),
-    userdb = require('./userdb'),
     UserError = user.UserError,
     _ = require('underscore');
 
@@ -27,11 +27,11 @@ function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     passport.serializeUser(function (user, callback) {
-        callback(null, user.username);
+        callback(null, user.id);
     });
 
-    passport.deserializeUser(function(username, callback) {
-        userdb.get(username, function (error, result) {
+    passport.deserializeUser(function(userId, callback) {
+        user.get(userId, function (error, result) {
             if (error) return callback(error);
 
             var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -43,7 +43,7 @@ function initialize(callback) {
 
     passport.use(new LocalStrategy(function (username, password, callback) {
         if (username.indexOf('@') === -1) {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                 if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                 if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                 if (error) return callback(error);
@@ -66,14 +66,14 @@ function initialize(callback) {
             debug('BasicStrategy: detected client id %s instead of username:password', username);
             // username is actually client id here
             // password is client secret
-            clientdb.get(username, function (error, client) {
-                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            clients.get(username, function (error, client) {
+                if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
                 if (client.clientSecret != password) return callback(null, false);
                 return callback(null, client);
             });
         } else {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                 if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                 if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                 if (error) return callback(error);
@@ -84,8 +84,8 @@ function initialize(callback) {
     }));
 
     passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function(error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) { return callback(error); }
             if (client.clientSecret != clientSecret) { return callback(null, false); }
             return callback(null, client);
@@ -100,29 +100,11 @@ function initialize(callback) {
             // scopes here can define what capabilities that token carries
             // passport put the 'info' object into req.authInfo, where we can further validate the scopes
             var info = { scope: token.scope };
-            var tokenType;
 
-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }
-
-            userdb.get(token.identifier, function (error, user) {
+            user.get(token.identifier, function (error, user) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
 
-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
                 callback(null, user, info);
             });
         });

src/backupdb.js

@@ -0,0 +1,114 @@
+'use strict';
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', ];
+
+exports = module.exports = {
+    add: add,
+    getPaged: getPaged,
+    get: get,
+    del: del,
+    getByAppIdPaged: getByAppIdPaged,
+
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+}
+
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, 'appbackup\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
+        [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function add(backup, callback) {
+    assert(backup && typeof backup === 'object');
+    assert.strictEqual(typeof backup.id, 'string');
+    assert.strictEqual(typeof backup.version, 'string');
+    assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
+    assert(util.isArray(backup.dependsOn));
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = backup.creationTime || new Date(); // allow tests to set the time
+
+    database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn) VALUES (?, ?, ?, ?, ?, ?)',
+        [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(',') ],
+        function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}

src/backups.js

@@ -3,21 +3,56 @@
 exports = module.exports = {
     BackupsError: BackupsError,
 
-    getAllPaged: getAllPaged,
+    getPaged: getPaged,
+    getByAppIdPaged: getByAppIdPaged,
 
-    getBackupUrl: getBackupUrl,
     getRestoreUrl: getRestoreUrl,
+    getRestoreConfig: getRestoreConfig,
 
-    copyLastBackup: copyLastBackup
+    ensureBackup: ensureBackup,
+
+    backup: backup,
+    backupApp: backupApp,
+    restoreApp: restoreApp,
+
+    backupBoxAndApps: backupBoxAndApps
 };
 
-var assert = require('assert'),
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    async = require('async'),
+    assert = require('assert'),
+    backupdb = require('./backupdb.js'),
     caas = require('./storage/caas.js'),
     config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:backups'),
+    eventlog = require('./eventlog.js'),
+    locker = require('./locker.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    progress = require('./progress.js'),
     s3 = require('./storage/s3.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
     settings = require('./settings.js'),
-    util = require('util');
+    superagent = require('superagent'),
+    util = require('util'),
+    webhooks = require('./webhooks.js');
+
+var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
 
 function BackupsError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
@@ -40,6 +75,7 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.BAD_STATE = 'bad state';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';
 
 // choose which storage backend we use for test purpose we use s3
@@ -51,49 +87,105 @@ function api(provider) {
     }
 }
 
-function getAllPaged(page, perPage, callback) {
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
     assert.strictEqual(typeof callback, 'function');
 
+    backupdb.getPaged(page, perPage, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    backupdb.getByAppIdPaged(page, perPage, appId, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getBoxBackupCredentials(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
+    var filename = filebase + '.tar.gz';
+
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
-            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
+            if (error) return callback(error);
 
-            return callback(null, backups); // [ { creationTime, restoreKey } ] sorted by time (latest first
+            result.id = filename;
+            result.s3Url = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + filename;
+            result.backupKey = backupConfig.key;
+
+            debug('getBoxBackupCredentials: %j', result);
+
+            callback(null, result);
         });
     });
 }
 
-function getBackupUrl(app, callback) {
-    assert(!app || typeof app === 'object');
+function getAppBackupCredentials(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var filename = '';
-    if (app) {
-        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    } else {
-        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
-    }
+    var now = new Date();
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
+    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
             if (error) return callback(error);
 
-            var obj = {
-                id: filename,
-                url: result.url,
-                sessionToken: result.sessionToken,
-                backupKey: backupConfig.key
-            };
+            result.id = dataFilename;
+            result.s3ConfigUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + configFilename;
+            result.s3DataUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + dataFilename;
+            result.backupKey = backupConfig.key;
 
-            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getAppBackupCredentials: %j', result);
 
-            callback(null, obj);
+            callback(null, result);
+        });
+    });
+}
+
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+function getRestoreConfig(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var configFile = backupId.replace(/\.tar\.gz$/, '.json');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getRestoreUrl(backupConfig, configFile, function (error, result) {
+            if (error) return callback(error);
+
+            superagent.get(result.url).buffer(true).end(function (error, response) {
+                if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+                if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
+
+                var config = safe.JSON.parse(response.text);
+                if (!config) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error in config:' + safe.error.message));
+
+                return callback(null, config);
+            });
         });
     });
 }
@@ -106,37 +198,299 @@ function getRestoreUrl(backupId, callback) {
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
+        api(backupConfig.provider).getRestoreUrl(backupConfig, backupId, function (error, result) {
             if (error) return callback(error);
 
             var obj = {
                 id: backupId,
                 url: result.url,
-                sessionToken: result.sessionToken,
                 backupKey: backupConfig.key
             };
 
-            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getRestoreUrl: id:%s url:%s backupKey:%s', obj.id, obj.url, obj.backupKey);
 
             callback(null, obj);
         });
     });
 }
 
-function copyLastBackup(app, callback) {
-    assert(app && typeof app === 'object');
+function copyLastBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifeset === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    var now = new Date();
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), manifest.version);
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilename, function (error) {
+        debug('copyLastBackup: copying archive %s to %s', app.lastBackupId, toFilenameArchive);
+
+        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
             if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
-            return callback(null, toFilename);
+            // TODO change that logic by adjusting app.lastBackupId to not contain the file type
+            var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';
+
+            debug('copyLastBackup: copying config %s to %s', configFileId, toFilenameConfig);
+
+            api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+                return callback(null, toFilenameArchive);
+            });
+        });
+    });
+}
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    getBoxBackupCredentials(appBackupIds, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        debug('backupBoxWithAppBackupIds:  %j', result);
+
+        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.sessionToken, result.region, result.backupKey ];
+
+        shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debug('backupBoxWithAppBackupIds: success');
+
+            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                    if (error) return callback(error);
+                    callback(null, result.id);
+                });
+            });
+        });
+    });
+}
+
+// this function expects you to have a lock
+// function backupBox(callback) {
+//    apps.getAll(function (error, allApps) {
+//         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+//
+//         var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+//         appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+//
+//         backupBoxWithAppBackupIds(appBackupIds, callback);
+//     });
+// }
+
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldAppBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    copyLastBackup(app, manifest, function (error, newBackupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewAppBackup(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAppBackupCredentials(app, manifest, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
+
+        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey,
+                     result.sessionToken, result.region, result.backupKey ];
+
+        async.series([
+            addons.backupAddons.bind(null, app, manifest.addons),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
+        ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debugApp(app, 'createNewAppBackup: %s done', result.id);
+
+            backupdb.add({ id: result.id, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, result.id);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function backupApp(app, manifest, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        backupFunction = reuseOldAppBackup.bind(null, app, manifest);
+    } else {
+        var appConfig = apps.getAppConfig(app);
+        appConfig.manifest = manifest;
+        backupFunction = createNewAppBackup.bind(null, app, manifest);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, function (error) {
+            if (error) return callback(error);
+
+            return callback(null, backupId);
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    callback = callback || NOOP_CALLBACK;
+
+    eventlog.add(eventlog.ACTION_BACKUP_START, auditSource, { });
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            backupApp(app, app.manifest, function (error, backupId) {
+                if (error && error.reason !== BackupsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
+                }
+
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+
+                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, filename: filename });
+
+                callback(error, filename);
+            });
+        });
+    });
+}
+
+function backup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
+
+    progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
+
+    backupBoxAndApps(auditSource, function (error) { // start the backup operation in the background
+        if (error) console.error('backup failed.', error);
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+
+    getPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        backup(auditSource, callback);
+    });
+}
+
+function restoreApp(app, addonsToRestore, backupId, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    getRestoreUrl(backupId, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
         });
     });
 }

src/cert/acme.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var assert = require('assert'),
@@ -7,6 +5,7 @@ var assert = require('assert'),
     crypto = require('crypto'),
     debug = require('debug')('box:cert/acme'),
     fs = require('fs'),
+    parseLinks = require('parse-links'),
     path = require('path'),
     paths = require('../paths.js'),
     safe = require('safetydance'),
@@ -58,7 +57,6 @@ function Acme(options) {
     this.caOrigin = options.prod ? CA_PROD : CA_STAGING;
     this.accountKeyPem = null; // Buffer
     this.email = options.email;
-    this.chainPem = options.prod ? safe.fs.readFileSync(__dirname + '/lets-encrypt-x1-cross-signed.pem.txt') : new Buffer('');
 }
 
 Acme.prototype.getNonce = function (callback) {
@@ -304,7 +302,7 @@ Acme.prototype.signCertificate = function (domain, csrDer, callback) {
 
         if (!certUrl) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Missing location in downloadCertificate'));
 
-        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // for renewal
+        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // maybe use for renewal
 
         return callback(null, result.headers.location);
     });
@@ -315,25 +313,57 @@ Acme.prototype.createKeyAndCsr = function (domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var outdir = paths.APP_CERTS_DIR;
+    var csrFile = path.join(outdir, domain + '.csr');
+    var privateKeyFile = path.join(outdir, domain + '.key');
     var execSync = safe.child_process.execSync;
 
-    var privateKeyFile = path.join(outdir, domain + '.key');
-    var key = execSync('openssl genrsa 4096');
-    if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (safe.fs.existsSync(privateKeyFile)) {
+        // in some old releases, csr file was corrupt. so always regenerate it
+        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
+    } else {
+        var key = execSync('openssl genrsa 4096');
+        if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
 
-    debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+    }
 
     var csrDer = execSync(util.format('openssl req -new -key %s -outform DER -subj /CN=%s', privateKeyFile, domain));
     if (!csrDer) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    var csrFile = path.join(outdir, domain + '.csr');
-    if (!safe.fs.writeFileSync(csrFile, csrFile)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error)); // bookkeeping
 
     debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
 
     callback(null, csrDer);
 };
 
+// TODO: download the chain in a loop following 'up' header
+Acme.prototype.downloadChain = function (linkHeader, callback) {
+    if (!linkHeader) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Empty link header when downloading certificate chain');
+
+    var linkInfo = parseLinks(linkHeader);
+    if (!linkInfo || !linkInfo.up) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Failed to parse link header when downloading certificate chain'); 
+
+    debug('downloadChain: downloading from %s', this.caOrigin + linkInfo.up);
+
+    superagent.get(this.caOrigin + linkInfo.up).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var chainDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        var chainPem = execSync('openssl x509 -inform DER -outform PEM', { input: chainDer }); // this is really just base64 encoding with header
+        if (!chainPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        callback(null, chainPem);
+    });
+};
+
 Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof certUrl, 'string');
@@ -355,18 +385,22 @@ Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
         var execSync = safe.child_process.execSync;
 
         safe.fs.writeFileSync(path.join(outdir, domain + '.der'), certificateDer);
-        debug('downloadCertificate: cert der file saved');
+        debug('downloadCertificate: cert der file for %s saved', domain);
 
         var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { input: certificateDer }); // this is really just base64 encoding with header
         if (!certificatePem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
 
-        var certificateFile = path.join(outdir, domain + '.cert');
-        var fullChainPem = Buffer.concat([certificatePem, that.chainPem]);
-        if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        that.downloadChain(result.header['link'], function (error, chainPem) {
+            if (error) return callback(error);
 
-        debug('downloadCertificate: cert file saved at %s', certificateFile);
+            var certificateFile = path.join(outdir, domain + '.cert');
+            var fullChainPem = Buffer.concat([certificatePem, chainPem]);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
 
-        callback();
+            debug('downloadCertificate: cert file for %s saved at %s', domain, certificateFile);
+
+            callback();
+        });
     });
 };
 
@@ -414,21 +448,11 @@ Acme.prototype.getCertificate = function (domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var outdir = paths.APP_CERTS_DIR;
-    var certUrl = safe.fs.readFileSync(path.join(outdir, domain + '.url'), 'utf8');
-    var certificateGetter;
-
-    if (certUrl) {
-        debug('getCertificate: renewing existing cert for %s from %s', domain, certUrl);
-        certificateGetter = this.downloadCertificate.bind(this, domain, certUrl);
-    } else {
-        debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
-        certificateGetter = this.acmeFlow.bind(this, domain);
-    }
-
-    certificateGetter(function (error) {
+    debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
+    this.acmeFlow(domain, function (error) {
         if (error) return callback(error);
 
+        var outdir = paths.APP_CERTS_DIR;
         callback(null, path.join(outdir, domain + '.cert'), path.join(outdir, domain + '.key'));
     });
 };

src/cert/lets-encrypt-x1-cross-signed.pem.txt

@@ -1,27 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEqDCCA5CgAwIBAgIRAJgT9HUT5XULQ+dDHpceRL0wDQYJKoZIhvcNAQELBQAw
-PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD
-Ew5EU1QgUm9vdCBDQSBYMzAeFw0xNTEwMTkyMjMzMzZaFw0yMDEwMTkyMjMzMzZa
-MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
-ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMTCCASIwDQYJKoZIhvcNAQEBBQAD
-ggEPADCCAQoCggEBAJzTDPBa5S5Ht3JdN4OzaGMw6tc1Jhkl4b2+NfFwki+3uEtB
-BaupnjUIWOyxKsRohwuj43Xk5vOnYnG6eYFgH9eRmp/z0HhncchpDpWRz/7mmelg
-PEjMfspNdxIknUcbWuu57B43ABycrHunBerOSuu9QeU2mLnL/W08lmjfIypCkAyG
-dGfIf6WauFJhFBM/ZemCh8vb+g5W9oaJ84U/l4avsNwa72sNlRZ9xCugZbKZBDZ1
-gGusSvMbkEl4L6KWTyogJSkExnTA0DHNjzE4lRa6qDO4Q/GxH8Mwf6J5MRM9LTb4
-4/zyM2q5OTHFr8SNDR1kFjOq+oQpttQLwNh9w5MCAwEAAaOCAZIwggGOMBIGA1Ud
-EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMH8GCCsGAQUFBwEBBHMwcTAy
-BggrBgEFBQcwAYYmaHR0cDovL2lzcmcudHJ1c3RpZC5vY3NwLmlkZW50cnVzdC5j
-b20wOwYIKwYBBQUHMAKGL2h0dHA6Ly9hcHBzLmlkZW50cnVzdC5jb20vcm9vdHMv
-ZHN0cm9vdGNheDMucDdjMB8GA1UdIwQYMBaAFMSnsaR7LHH62+FLkHX/xBVghYkQ
-MFQGA1UdIARNMEswCAYGZ4EMAQIBMD8GCysGAQQBgt8TAQEBMDAwLgYIKwYBBQUH
-AgEWImh0dHA6Ly9jcHMucm9vdC14MS5sZXRzZW5jcnlwdC5vcmcwPAYDVR0fBDUw
-MzAxoC+gLYYraHR0cDovL2NybC5pZGVudHJ1c3QuY29tL0RTVFJPT1RDQVgzQ1JM
-LmNybDATBgNVHR4EDDAKoQgwBoIELm1pbDAdBgNVHQ4EFgQUqEpqYwR93brm0Tm3
-pkVl7/Oo7KEwDQYJKoZIhvcNAQELBQADggEBANHIIkus7+MJiZZQsY14cCoBG1hd
-v0J20/FyWo5ppnfjL78S2k4s2GLRJ7iD9ZDKErndvbNFGcsW+9kKK/TnY21hp4Dd
-ITv8S9ZYQ7oaoqs7HwhEMY9sibED4aXw09xrJZTC9zK1uIfW6t5dHQjuOWv+HHoW
-ZnupyxpsEUlEaFb+/SCI4KCSBdAsYxAcsHYI5xxEI4LutHp6s3OT2FuO90WfdsIk
-6q78OMSdn875bNjdBYAqxUp2/LEIHfDBkLoQz0hFJmwAbYahqKaLn73PAAm1X2kj
-f1w8DdnkabOLGeOVcj9LQ+s67vBykx4anTjURkbqZslUEUsn2k5xeua2zUk=
------END CERTIFICATE-----

src/certificates.js

@@ -1,8 +1,18 @@
-/* jslint node:true */
-
 'use strict';
 
+exports = module.exports = {
+    installAdminCertificate: installAdminCertificate,
+    renewAll: renewAll,
+    setFallbackCertificate: setFallbackCertificate,
+    setAdminCertificate: setAdminCertificate,
+    CertificatesError: CertificatesError,
+    validateCertificate: validateCertificate,
+    ensureCertificate: ensureCertificate,
+    getAdminCertificatePath: getAdminCertificatePath
+};
+
 var acme = require('./cert/acme.js'),
+    apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     caas = require('./cert/caas.js'),
@@ -10,7 +20,9 @@ var acme = require('./cert/acme.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:src/certificates'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
+    mailer = require('./mailer.js'),
     nginx = require('./nginx.js'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -22,16 +34,6 @@ var acme = require('./cert/acme.js'),
     waitForDns = require('./waitfordns.js'),
     x509 = require('x509');
 
-exports = module.exports = {
-    installAdminCertificate: installAdminCertificate,
-    autoRenew: autoRenew,
-    setFallbackCertificate: setFallbackCertificate,
-    setAdminCertificate: setAdminCertificate,
-    CertificatesError: CertificatesError,
-    validateCertificate: validateCertificate,
-    ensureCertificate: ensureCertificate
-};
-
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 function CertificatesError(reason, errorOrMessage) {
@@ -55,15 +57,20 @@ function CertificatesError(reason, errorOrMessage) {
 util.inherits(CertificatesError, Error);
 CertificatesError.INTERNAL_ERROR = 'Internal Error';
 CertificatesError.INVALID_CERT = 'Invalid certificate';
+CertificatesError.NOT_FOUND = 'Not Found';
+
+function getApi(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-function getApi(callback) {
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
-        var api = tlsConfig.provider === 'caas' ? caas : acme;
+        var api = !app.altDomain && tlsConfig.provider === 'caas' ? caas : acme;
 
         var options = { };
-        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        // used by acme backend to determine the LE origin.
+        options.prod = (api === caas) ? !config.isDev() : tlsConfig.provider.match(/.*-prod/) !== null;
 
         // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
         // we cannot use admin@fqdn because the user might not have set it up.
@@ -88,10 +95,10 @@ function installAdminCertificate(callback) {
         sysinfo.getIp(function (error, ip) {
             if (error) return callback(error);
 
-            waitForDns(config.adminFqdn(), ip, config.fqdn(), function (error) {
+            waitForDns(config.adminFqdn(), ip, 'A', function (error) {
                 if (error) return callback(error); // this cannot happen because we retry forever
 
-                ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
+                ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
                     if (error) { // currently, this can never happen
                         debug('Error obtaining certificate. Proceed anyway', error);
                         return callback();
@@ -104,41 +111,89 @@ function installAdminCertificate(callback) {
     });
 }
 
-function needsRenewalSync(certFilePath) {
-    var result = safe.child_process.execSync('openssl x509 -checkend %s -in %s', 60 * 60 * 24 * 5, certFilePath);
+function isExpiringSync(certFilePath, hours) {
+    assert.strictEqual(typeof certFilePath, 'string');
+    assert.strictEqual(typeof hours, 'number');
 
-    return result === null; // command errored
+    if (!fs.existsSync(certFilePath)) return 2; // not found
+
+    var result = safe.child_process.spawnSync('/usr/bin/openssl', [ 'x509', '-checkend', String(60 * 60 * hours), '-in', certFilePath ]);
+
+    debug('isExpiringSync: %s %s %s', certFilePath, result.stdout.toString('utf8').trim(), result.status);
+
+    return result.status === 1; // 1 - expired 0 - not expired
 }
 
-function autoRenew(callback) {
-    debug('autoRenew: Checking certificates for renewal');
-    callback = callback || NOOP_CALLBACK;
+function renewAll(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    var filenames = safe.fs.readdirSync(paths.APP_CERTS_DIR);
-    if (!filenames) {
-        debug('autoRenew: Error getting filenames: %s', safe.error.message);
-        return;
-    }
+    debug('renewAll: Checking certificates for renewal');
 
-    var certs = filenames.filter(function (f) {
-        return f.match(/\.cert$/) !== null && needsRenewalSync(path.join(paths.APP_CERTS_DIR, f));
-    });
-
-    debug('autoRenew: %j needs to be renewed', certs);
-
-    getApi(function (error, api, apiOptions) {
+    apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        async.eachSeries(certs, function iterator(cert, iteratorCallback) {
-            var domain = cert.match(/^(.*)\.cert$/)[1];
-            if (domain === 'host') return iteratorCallback(); // cannot renew fallback cert
+        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app
 
-            debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+        var expiringApps = [ ];
+        for (var i = 0; i < allApps.length; i++) {
+            var appDomain = allApps[i].altDomain || config.appFqdn(allApps[i].location);
+            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
+            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
-            api.getCertificate(domain, apiOptions, function (error) {
-                if (error) debug('autoRenew: could not renew cert for %s', domain, error);
+            if (!safe.fs.existsSync(keyFilePath)) {
+                debug('renewAll: no existing key file for %s. skipping', appDomain);
+                continue;
+            }
 
-                iteratorCallback(); // move on to next cert
+            if (isExpiringSync(certFilePath, 24 * 30)) { // expired or not found
+                expiringApps.push(allApps[i]);
+            }
+        }
+
+        debug('renewAll: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
+
+        async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+            var domain = app.altDomain || config.appFqdn(app.location);
+
+            getApi(app, function (error, api, apiOptions) {
+                if (error) return callback(error);
+
+                debug('renewAll: renewing cert for %s with options %j', domain, apiOptions);
+
+                api.getCertificate(domain, apiOptions, function (error) {
+                    var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+                    var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+                    var errorMessage = error ? error.message : '';
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
+                    mailer.certificateRenewed(domain, errorMessage);
+
+                    if (error) {
+                        debug('renewAll: could not renew cert for %s because %s', domain, error);
+
+                        // check if we should fallback if we expire in the coming day
+                        if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
+
+                        debug('renewAll: using fallback certs for %s since it expires soon', domain, error);
+
+                        certFilePath = 'cert/host.cert';
+                        keyFilePath = 'cert/host.key';
+                    } else {
+                        debug('renewAll: certificate for %s renewed', domain);
+                    }
+
+                    // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
+                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                        nginx.configureAdmin.bind(null, certFilePath, keyFilePath)
+                        : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
+
+                    configureFunc(function (ignoredError) {
+                        if (ignoredError) debug('fallbackExpiredCertificates: error reconfiguring app', ignoredError);
+
+                        iteratorCallback(); // move to next app
+                    });
+                });
             });
         });
     });
@@ -207,12 +262,20 @@ function setFallbackCertificate(cert, key, callback) {
     });
 }
 
+function getFallbackCertificatePath(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // any user fallback cert is always copied over to nginx cert dir
+    callback(null, path.join(paths.NGINX_CERT_DIR, 'host.cert'), path.join(paths.NGINX_CERT_DIR, 'host.key'));
+}
+
+// FIXME: setting admin cert needs to restart the mail container because it uses admin cert
 function setAdminCertificate(cert, key, callback) {
     assert.strictEqual(typeof cert, 'string');
     assert.strictEqual(typeof key, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
+    var vhost = config.adminFqdn();
     var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
     var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
 
@@ -226,10 +289,24 @@ function setAdminCertificate(cert, key, callback) {
     nginx.configureAdmin(certFilePath, keyFilePath, callback);
 }
 
-function ensureCertificate(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
+function getAdminCertificatePath(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    var vhost = config.adminFqdn();
+    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, certFilePath, keyFilePath);
+
+    getFallbackCertificatePath(callback);
+}
+
+function ensureCertificate(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var domain = app.altDomain || config.appFqdn(app.location);
+
     // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
     var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
     var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
@@ -237,12 +314,12 @@ function ensureCertificate(domain, callback) {
     if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
         debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);
 
-        if (!needsRenewalSync(userCertFilePath)) return callback(null, userCertFilePath, userKeyFilePath);
-
-        debug('ensureCertificate: %s cert require renewal', domain);
+        if (!isExpiringSync(userCertFilePath, 24 * 1)) return callback(null, userCertFilePath, userKeyFilePath);
     }
 
-    getApi(function (error, api, apiOptions) {
+    debug('ensureCertificate: %s cert require renewal', domain);
+
+    getApi(app, function (error, api, apiOptions) {
         if (error) return callback(error);
 
         debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);

src/clientdb.js

@@ -5,6 +5,7 @@
 exports = module.exports = {
     get: get,
     getAll: getAll,
+    getAllWithTokenCount: getAllWithTokenCount,
     getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
@@ -14,13 +15,7 @@ exports = module.exports = {
     delByAppId: delByAppId,
     delByAppIdAndType: delByAppIdAndType,
 
-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    _clear: clear
 };
 
 var assert = require('assert'),
@@ -52,14 +47,24 @@ function getAll(callback) {
     });
 }
 
+function getAllWithTokenCount(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
 function getAllWithTokenCountByIdentifier(identifier, callback) {
     assert.strictEqual(typeof identifier, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        callback(null, results);
 
+        callback(null, results);
     });
 }
 
@@ -71,7 +76,7 @@ function getByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -84,7 +89,7 @@ function getByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -127,7 +132,7 @@ function delByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
@@ -140,17 +145,17 @@ function delByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.query('DELETE FROM clients WHERE id!="cid-webadmin" AND id!="cid-sdk" AND id!="cid-cli"', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null);
+        callback(null);
     });
 }
 

src/clients.js

@@ -6,9 +6,32 @@ exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getAll: getAll,
+    getByAppIdAndType: getByAppIdAndType,
     getClientTokensByUserId: getClientTokensByUserId,
-    delClientTokensByUserId: delClientTokensByUserId
+    delClientTokensByUserId: delClientTokensByUserId,
+    delByAppIdAndType: delByAppIdAndType,
+    addClientTokenByUserId: addClientTokenByUserId,
+    delToken: delToken,
+
+    // keep this in sync with start.sh ADMIN_SCOPES that generates the cid-webadmin
+    SCOPE_APPS: 'apps',
+    SCOPE_DEVELOPER: 'developer',
+    SCOPE_PROFILE: 'profile',
+    SCOPE_CLOUDRON: 'cloudron',
+    SCOPE_SETTINGS: 'settings',
+    SCOPE_USERS: 'users',
+
+    // roles are handled just like the above scopes, they are parallel to scopes
+    // scopes enclose API groups, roles specify the usage role
+    SCOPE_ROLE_SDK: 'roleSdk',
+
+    // client type enums
+    TYPE_EXTERNAL: 'external',
+    TYPE_BUILT_IN: 'built-in',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy'
 };
 
 var assert = require('assert'),
@@ -16,7 +39,6 @@ var assert = require('assert'),
     hat = require('hat'),
     appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
-    constants = require('./constants.js'),
     async = require('async'),
     clientdb = require('./clientdb.js'),
     DatabaseError = require('./databaseerror.js'),
@@ -43,14 +65,29 @@ function ClientsError(reason, errorOrMessage) {
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
 ClientsError.INVALID_CLIENT = 'Invalid client';
+ClientsError.INVALID_TOKEN = 'Invalid token';
+ClientsError.NOT_FOUND = 'Not found';
+ClientsError.INTERNAL_ERROR = 'Internal Error';
+ClientsError.NOT_ALLOWED = 'Not allowed to remove this client';
 
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
 
-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
+    var VALID_SCOPES = [
+        exports.SCOPE_APPS,
+        exports.SCOPE_DEVELOPER,
+        exports.SCOPE_PROFILE,
+        exports.SCOPE_CLOUDRON,
+        exports.SCOPE_SETTINGS,
+        exports.SCOPE_USERS,
+        '*',    // includes all scopes, but not roles
+        exports.SCOPE_ROLE_SDK
+    ];
 
-    // TODO maybe validate all individual scopes if they exist
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE, 'Empty scope not allowed');
+
+    var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE, 'Invalid scope. Available scopes are ' + VALID_SCOPES.join(', '));
 
     return null;
 }
@@ -62,6 +99,9 @@ function add(appId, type, redirectURI, scope, callback) {
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    // allow whitespace
+    scope = scope.split(',').map(function (s) { return s.trim(); }).join(',');
+
     var error = validateScope(scope);
     if (error) return callback(error);
 
@@ -89,6 +129,7 @@ function get(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
@@ -99,24 +140,24 @@ function del(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.del(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
 }
 
-function getAllWithDetailsByUserId(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
+function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+    clientdb.getAll(function (error, results) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
         if (error) return callback(error);
 
         var tmp = [];
         async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
-                record.name = constants.ADMIN_NAME;
-                record.location = constants.ADMIN_LOCATION;
+            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
+                // the appId in this case holds the name
+                record.name = record.appId;
 
                 tmp.push(record);
 
@@ -125,14 +166,13 @@ function getAllWithDetailsByUserId(userId, callback) {
 
             appdb.get(record.appId, function (error, result) {
                 if (error) {
-                    console.error('Failed to get app details for oauth client', result, error);
+                    console.error('Failed to get app details for oauth client', record.appId, error);
                     return callback(null);  // ignore error so we continue listing clients
                 }
 
-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === exports.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
 
                 record.location = result.location;
 
@@ -147,15 +187,27 @@ function getAllWithDetailsByUserId(userId, callback) {
     });
 }
 
+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getByAppIdAndType(appId, type, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
 function getClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof clientId, 'string');
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null, []);
             });
@@ -171,10 +223,10 @@ function delClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null);
             });
@@ -184,3 +236,57 @@ function delClientTokensByUserId(clientId, userId, callback) {
         callback(null);
     });
 }
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(appId, type, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null);
+    });
+}
+
+function addClientTokenByUserId(clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        var token = tokendb.generateToken();
+
+        tokendb.add(token, userId, result.id, expiresAt, result.scope, function (error) {
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                accessToken: token,
+                identifier: userId,
+                clientId: result.id,
+                scope: result.id,
+                expires: expiresAt
+            });
+        });
+    });
+}
+
+function delToken(clientId, tokenId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof tokenId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        tokendb.del(tokenId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.INVALID_TOKEN, 'Invalid token'));
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
+    });
+}

src/cloudron.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -14,12 +12,8 @@ exports = module.exports = {
     sendHeartbeat: sendHeartbeat,
 
     updateToLatest: updateToLatest,
-    update: update,
     reboot: reboot,
-    migrate: migrate,
-    backup: backup,
     retire: retire,
-    ensureBackup: ensureBackup,
 
     isConfiguredSync: isConfiguredSync,
 
@@ -28,19 +22,20 @@ exports = module.exports = {
     events: new (require('events').EventEmitter)(),
 
     EVENT_ACTIVATED: 'activated',
-    EVENT_CONFIGURED: 'configured'
+    EVENT_CONFIGURED: 'configured',
+    EVENT_FIRST_RUN: 'firstrun'
 };
 
 var apps = require('./apps.js'),
-    AppsError = require('./apps.js').AppsError,
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:cloudron'),
     df = require('node-df'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     locker = require('./locker.js'),
     mailer = require('./mailer.js'),
@@ -58,13 +53,10 @@ var apps = require('./apps.js'),
     updateChecker = require('./updatechecker.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    userdb = require('./userdb.js'),
-    util = require('util'),
-    webhooks = require('./webhooks.js');
+    user = require('./user.js'),
+    util = require('util');
 
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
     RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');
 
@@ -72,24 +64,9 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 var gUpdatingDns = false,                // flag for dns update reentrancy
     gCloudronDetails = null,             // cached cloudron details like region,size...
+    gAppstoreUserDetails = {},
     gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
 
-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 function CloudronError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -113,10 +90,6 @@ CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
 CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
 CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
@@ -124,15 +97,36 @@ CloudronError.NOT_FOUND = 'Not found';
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    ensureDkimKeySync();
 
-    syncConfigState(callback);
+    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.on(exports.EVENT_FIRST_RUN, installAppBundle);
+
+    // check activation state for existing cloudrons that do not have first run file
+    // can be removed once cloudrons have been updated
+    isActivated(function (error, activated) {
+        if (error) return callback(error);
+
+        debug('initialize: cloudron %s activated', activated ? '' : 'not');
+
+        if (activated) fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+
+        if (!fs.existsSync(paths.FIRST_RUN_FILE)) {
+            // EE API is sync. do not keep the server waiting
+            debug('initialize: emitting first run event');
+            process.nextTick(function () { exports.events.emit(exports.EVENT_FIRST_RUN); });
+            fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+        }
+
+        syncConfigState(callback);
+    });
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.removeListener(exports.EVENT_FIRST_RUN, installAppBundle);
 
     callback(null);
 }
@@ -141,6 +135,15 @@ function isConfiguredSync() {
     return gIsConfigured === true;
 }
 
+function isActivated(callback) {
+    user.getOwner(function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+        if (error) return callback(error);
+
+        callback(null, true);
+    });
+}
+
 function isConfigured(callback) {
     // set of rules to see if we have the configs required for cloudron to function
     // note this checks for missing configs and not invalid configs
@@ -185,55 +188,62 @@ function setTimeZone(ip, callback) {
 
     debug('setTimeZone ip:%s', ip);
 
-    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
+    // https://github.com/bluesmoon/node-geoip
+    // https://github.com/runk/node-maxmind
+    // { url: 'http://freegeoip.net/json/%s', jpath: 'time_zone' },
+    // { url: 'http://ip-api.com/json/%s', jpath: 'timezone' },
+    // { url: 'http://geoip.nekudo.com/api/%s', jpath: 'time_zone }
+
+    superagent.get('http://freegeoip.net/json/' + ip).end(function (error, result) {
         if ((error && !error.response) || result.statusCode !== 200) {
             debug('Failed to get geo location: %s', error.message);
             return callback(null);
         }
 
-        if (!result.body.timezone) {
+        if (!result.body.time_zone || typeof result.body.time_zone !== 'string') {
             debug('No timezone in geoip response : %j', result.body);
             return callback(null);
         }
 
-        debug('Setting timezone to ', result.body.timezone);
+        debug('Setting timezone to ', result.body.time_zone);
 
-        settings.setTimeZone(result.body.timezone, callback);
+        settings.setTimeZone(result.body.time_zone, callback);
     });
 }
 
-function activate(username, password, email, displayName, ip, callback) {
+function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof displayName, 'string');
     assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('activating user:%s email:%s', username, email);
 
     setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
-    user.createOwner(username, password, email, displayName, function (error, userObject) {
+    user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error.reason === UserError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clients.get('cid-webadmin', function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             // Also generate a token so the admin creation can also act as a login
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
                 // EE API is sync. do not keep the REST API reponse waiting
                 process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });
 
+                eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
+
                 callback(null, { token: token, expires: expires });
             });
         });
@@ -243,7 +253,7 @@ function activate(username, password, email, displayName, ip, callback) {
 function getStatus(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.count(function (error, count) {
+    user.count(function (error, count) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         settings.getCloudronName(function (error, cloudronName) {
@@ -283,6 +293,7 @@ function getCloudronDetails(callback) {
             if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
 
             gCloudronDetails = result.body.box;
+            gAppstoreUserDetails = result.body.user;
 
             return callback(null, gCloudronDetails);
         });
@@ -324,6 +335,7 @@ function getConfig(callback) {
                         developerMode: developerMode,
                         region: result.region,
                         size: result.size,
+                        billing: !!gAppstoreUserDetails.billing,
                         memory: os.totalmem(),
                         provider: config.provider(),
                         cloudronName: cloudronName
@@ -345,6 +357,21 @@ function sendHeartbeat() {
     });
 }
 
+function ensureDkimKeySync() {
+    var dkimPrivateKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/private');
+    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
+
+    if (fs.existsSync(dkimPrivateKeyFile) && fs.existsSync(dkimPublicKeyFile)) {
+        debug('DKIM keys already present');
+        return;
+    }
+
+    debug('Generating new DKIM keys');
+
+    safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024');
+    safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM');
+}
+
 function readDkimPublicKeySync() {
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
@@ -360,6 +387,7 @@ function readDkimPublicKeySync() {
     return publicKey;
 }
 
+// NOTE: if you change the SPF record here, be sure the wait check in mailer.js
 function txtRecordsWithSpf(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -373,16 +401,16 @@ function txtRecordsWithSpf(callback) {
         for (i = 0; i < txtRecords.length; i++) {
             if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF
 
-            validSpf = txtRecords[i].indexOf(' a:' + config.fqdn() + ' ') !== -1;
+            validSpf = txtRecords[i].indexOf(' a:' + config.adminFqdn() + ' ') !== -1;
             break;
         }
 
         if (validSpf) return callback(null, null);
 
         if (i == txtRecords.length) {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ~all"';
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ~all"';
         } else {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
         }
 
         return callback(null, txtRecords);
@@ -404,27 +432,33 @@ function addDnsRecords() {
     var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
 
     var dkimKey = readDkimPublicKeySync();
-    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
+    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
 
     sysinfo.getIp(function (error, ip) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
-        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
+        var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
         // DMARC requires special setup if report email id is in different domain
         var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
 
+        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };
+
         var records = [ ];
         if (config.isCustomDomain()) {
             records.push(webadminRecord);
             records.push(dkimRecord);
+            records.push(mxRecord);
         } else {
+            // for custom domains, we show a nakeddomain.html page
+            var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
+
             records.push(nakedDomainRecord);
             records.push(webadminRecord);
             records.push(dkimRecord);
             records.push(dmarcRecord);
+            records.push(mxRecord);
         }
 
         debug('addDnsRecords: %j', records);
@@ -460,51 +494,9 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function migrate(size, region, callback) {
-    assert.strictEqual(typeof size, 'string');
-    assert.strictEqual(typeof region, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_MIGRATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    function unlock(error) {
-        if (error) {
-            debug('Failed to migrate', error);
-            locker.unlock(locker.OP_MIGRATE);
-        } else {
-            debug('Migration initiated successfully');
-            // do not unlock; cloudron is migrating
-        }
-
-        return;
-    }
-
-    // initiate the migration in the background
-    backupBoxAndApps(function (error, restoreKey) {
-        if (error) return unlock(error);
-
-        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
-
-        superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send({ size: size, region: region, restoreKey: restoreKey })
-          .end(function (error, result) {
-            if (error && !error.response) return unlock(error);
-            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
-
-            return unlock(null);
-        });
-    });
-
-    callback(null);
-}
-
-function update(boxUpdateInfo, callback) {
+function update(boxUpdateInfo, auditSource, callback) {
     assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (!boxUpdateInfo) return callback(null);
@@ -512,6 +504,8 @@ function update(boxUpdateInfo, callback) {
     var error = locker.lock(locker.OP_BOX_UPDATE);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
     // ensure tools can 'wait' on progress
     progress.set(progress.UPDATE, 0, 'Starting');
 
@@ -543,13 +537,14 @@ function update(boxUpdateInfo, callback) {
 }
 
 
-function updateToLatest(callback) {
+function updateToLatest(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var boxUpdateInfo = updateChecker.getUpdateInfo().box;
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
 
-    update(boxUpdateInfo, callback);
+    update(boxUpdateInfo, auditSource, callback);
 }
 
 function doShortCircuitUpdate(boxUpdateInfo, callback) {
@@ -572,7 +567,7 @@ function doUpgrade(boxUpdateInfo, callback) {
 
     progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
 
-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'upgrader' }, function (error) {
         if (error) return upgradeError(error);
 
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
@@ -601,7 +596,7 @@ function doUpdate(boxUpdateInfo, callback) {
 
     progress.set(progress.UPDATE, 5, 'Backing up for update');
 
-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
         if (error) return updateError(error);
 
         // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
@@ -648,119 +643,31 @@ function doUpdate(boxUpdateInfo, callback) {
     });
 }
 
-function backup(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_FULL_BACKUP);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    // ensure tools can 'wait' on progress
-    progress.set(progress.BACKUP, 0, 'Starting');
-
-    // start the backup operation in the background
-    backupBoxAndApps(function (error) {
-        if (error) console.error('backup failed.', error);
-
-        locker.unlock(locker.OP_FULL_BACKUP);
-    });
-
-    callback(null);
-}
-
-function ensureBackup(callback) {
+function installAppBundle(callback) {
     callback = callback || NOOP_CALLBACK;
 
-    backups.getAllPaged(1, 1, function (error, backups) {
-        if (error) {
-            debug('Unable to list backups', error);
-            return callback(error); // no point trying to backup if appstore is down
-        }
+    var bundle = config.get('appBundle');
 
-        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
-            debug('Previous backup was %j, no need to backup now', backups[0]);
-            return callback(null);
-        }
+    if (!bundle || bundle.length === 0) {
+        debug('installAppBundle: no bundle set');
+        return callback();
+    }
 
-        backup(callback);
-    });
-}
+    async.eachSeries(bundle, function (appInfo, iteratorCallback) {
+        debug('autoInstall: installing %s at %s', appInfo.appstoreId, appInfo.location);
 
-function backupBoxWithAppBackupIds(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
+        var data = {
+            appStoreId: appInfo.appstoreId,
+            location: appInfo.location,
+            portBindings: appInfo.portBindings || null,
+            accessRestriction: appInfo.accessRestriction || null,
+        };
 
-    backups.getBackupUrl(null /* app */, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+        apps.install(data, { userId: null, username: 'autoinstaller' }, iteratorCallback);
+    }, function (error) {
+        if (error) debug('autoInstallApps: ', error);
 
-        debug('backup: url %s', result.url);
-
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            debug('backup: successful');
-
-            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                if (error) return callback(error);
-                callback(null, result.id);
-            });
-        });
-    });
-}
-
-// this function expects you to have a lock
-function backupBox(callback) {
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
-
-        backupBoxWithAppBackupIds(appBackupIds, callback);
-    });
-}
-
-// this function expects you to have a lock
-function backupBoxAndApps(callback) {
-    callback = callback || NOOP_CALLBACK;
-
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var processed = 0;
-        var step = 100/(allApps.length+1);
-
-        progress.set(progress.BACKUP, processed, '');
-
-        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
-            ++processed;
-
-            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                if (error && error.reason !== AppsError.BAD_STATE) {
-                    debugApp(app, 'Unable to backup', error);
-                    return iteratorCallback(error);
-                }
-
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
-                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
-            });
-        }, function appsBackedUp(error, backupIds) {
-            if (error) {
-                progress.set(progress.BACKUP, 100, error.message);
-                return callback(error);
-            }
-
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
-
-            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
-                progress.set(progress.BACKUP, 100, error ? error.message : '');
-                callback(error, restoreKey);
-            });
-        });
+        callback();
     });
 }
 

src/collectd.config.ejs

@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.stat">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
     Instance "<%= appId %>-memory"
     Separator " \\n"
     <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
     Instance "<%= appId %>-memory"
     Separator "\\n"
     <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker/<%= containerId %>/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
     Instance "<%= appId %>-cpu"
     Separator " \\n"
     <Result>

src/config.js

@@ -1,11 +1,7 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
     baseDir: baseDir,
-    dnsInSync: dnsInSync,
-    setDnsInSync: setDnsInSync,
 
     // values set here will be lost after a upgrade/update. use the sqlite database
     // for persistent values that need to be backed up
@@ -30,14 +26,16 @@ exports = module.exports = {
     // these values are derived
     adminOrigin: adminOrigin,
     internalAdminOrigin: internalAdminOrigin,
+    sysadminOrigin: sysadminOrigin, // caas routes
     adminFqdn: adminFqdn,
+    mailFqdn: mailFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
 
     isDev: isDev,
 
     // for testing resets to defaults
-    _reset: initConfig
+    _reset: _reset
 };
 
 var assert = require('assert'),
@@ -58,33 +56,34 @@ function baseDir() {
 
 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
 
-function dnsInSync() {
-    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
-}
-
-function setDnsInSync(content) {
-    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
-}
-
 function saveSync() {
     fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
 
+function _reset (callback) {
+    safe.fs.unlinkSync(cloudronConfigFileName);
+
+    initConfig();
+
+    if (callback) callback();
+}
+
 function initConfig() {
     // setup defaults
     data.fqdn = 'localhost';
 
     data.token = null;
-    data.adminEmail = null;
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
-    data.internalPort = 3001;
+    data.smtpPort = 2525; // // this value comes from mail container
+    data.sysadminPort = 3001;
     data.ldapPort = 3002;
     data.oauthProxyPort = 3003;
     data.simpleAuthPort = 3004;
     data.provider = 'caas';
+    data.appBundle = [ ];
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -101,7 +100,6 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
-        data.adminEmail = 'test@cloudron.foo';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -164,6 +162,10 @@ function adminFqdn() {
     return appFqdn(constants.ADMIN_LOCATION);
 }
 
+function mailFqdn() {
+    return appFqdn(constants.MAIL_LOCATION);
+}
+
 function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
@@ -172,6 +174,10 @@ function internalAdminOrigin() {
     return 'http://127.0.0.1:' + get('port');
 }
 
+function sysadminOrigin() {
+    return 'http://127.0.0.1:' + get('sysadminPort');
+}
+
 function token() {
     return get('token');
 }

src/constants.js

@@ -4,9 +4,16 @@
 exports = module.exports = {
     ADMIN_LOCATION: 'my',
     API_LOCATION: 'api', // this is unused but reserved for future use (#403)
+    SMTP_LOCATION: 'smtp',
+    IMAP_LOCATION: 'imap',
+    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
+    POSTMAN_LOCATION: 'postman', // used in dovecot bounces
+
     ADMIN_NAME: 'Settings',
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
-    ADMIN_APPID: 'admin' // admin appid (settingsdb)
+    ADMIN_APPID: 'admin', // admin appid (settingsdb)
+
+    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
 };
 

src/cron.js

@@ -7,6 +7,7 @@ exports = module.exports = {
 
 var apps = require('./apps.js'),
     assert = require('assert'),
+    backups = require('./backups.js'),
     certificates = require('./certificates.js'),
     cloudron = require('./cloudron.js'),
     config = require('./config.js'),
@@ -29,6 +30,7 @@ var gAutoupdaterJob = null,
     gCheckDiskSpaceJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var AUDIT_SOURCE = { userId: null, username: 'cron' };
 
 // cron format
 // Seconds: 0-59
@@ -65,7 +67,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
             cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: cloudron.ensureBackup,
+            onTick: backups.ensureBackup.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -121,7 +123,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gCertificateRenewJob) gCertificateRenewJob.stop();
         gCertificateRenewJob = new CronJob({
             cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: certificates.autoRenew,
+            onTick: certificates.renewAll.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -153,10 +155,10 @@ function autoupdatePatternChanged(pattern) {
             var updateInfo = updateChecker.getUpdateInfo();
             if (updateInfo.box) {
                 debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
             } else if (updateInfo.apps) {
                 debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+                apps.updateApps(updateInfo.apps, AUDIT_SOURCE, NOOP_CALLBACK);
             } else {
                 debug('No auto updates available');
             }

src/database.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -116,10 +114,14 @@ function clear(callback) {
     async.series([
         require('./appdb.js')._clear,
         require('./authcodedb.js')._clear,
+        require('./backupdb.js')._clear,
         require('./clientdb.js')._clear,
         require('./tokendb.js')._clear,
+        require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear
+        require('./settingsdb.js')._clear,
+        require('./eventlogdb.js')._clear,
+        require('./mailboxdb.js')._clear
     ], callback);
 }
 

src/databaseerror.js

@@ -30,3 +30,4 @@ DatabaseError.INTERNAL_ERROR = 'Internal error';
 DatabaseError.ALREADY_EXISTS = 'Entry already exist';
 DatabaseError.NOT_FOUND = 'Record not found';
 DatabaseError.BAD_FIELD = 'Invalid field';
+DatabaseError.IN_USE = 'In Use';

src/developer.js

@@ -5,7 +5,7 @@
 exports = module.exports = {
     DeveloperError: DeveloperError,
 
-    enabled: enabled,
+    isEnabled: isEnabled,
     setEnabled: setEnabled,
     issueDeveloperToken: issueDeveloperToken,
     getNonApprovedApps: getNonApprovedApps
@@ -13,7 +13,9 @@ exports = module.exports = {
 
 var assert = require('assert'),
     config = require('./config.js'),
+    clients = require('./clients.js'),
     debug = require('debug')('box:developer'),
+    eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
     settings = require('./settings.js'),
     superagent = require('superagent'),
@@ -41,7 +43,7 @@ util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
 DeveloperError.EXTERNAL_ERROR = 'External Error';
 
-function enabled(callback) {
+function isEnabled(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getDeveloperMode(function (error, enabled) {
@@ -50,27 +52,35 @@ function enabled(callback) {
     });
 }
 
-function setEnabled(enabled, callback) {
+function setEnabled(enabled, auditSource, callback) {
     assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     settings.setDeveloperMode(enabled, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        eventlog.add(eventlog.ACTION_CLI_MODE, auditSource, { enabled: enabled });
+
         callback(null);
     });
 }
 
-function issueDeveloperToken(user, callback) {
+function issueDeveloperToken(user, auditSource, callback) {
     assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var scopes = '*,' + clients.SCOPE_ROLE_SDK;
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
+    tokendb.add(token, user.id, 'cid-cli', expiresAt, scopes, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
-        callback(null, { token: token, expiresAt: expiresAt });
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
+
+        callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
     });
 }
 

src/dns/route53.js

@@ -39,7 +39,8 @@ function getZoneByName(dnsConfig, zoneName, callback) {
 
     var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
     route53.listHostedZones({}, function (error, result) {
-        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
 
         var zone = result.HostedZones.filter(function (zone) {
             return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
@@ -84,11 +85,9 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
 
         var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
         route53.changeResourceRecordSets(params, function(error, result) {
-            if (error && error.code === 'PriorRequestNotComplete') {
-                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
-            } else if (error) {
-                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
-            }
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error && error.code === 'PriorRequestNotComplete') return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
 
             callback(null, result.ChangeInfo.Id);
         });
@@ -131,7 +130,8 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {
 
         var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
         route53.listResourceRecordSets(params, function (error, result) {
-            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
             if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
             if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
 
@@ -175,21 +175,22 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
 
         var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
         route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
             if (error && error.message && error.message.indexOf('it was not found') !== -1) {
                 debug('del: resource record set not found.', error);
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
             } else if (error && error.code === 'NoSuchHostedZone') {
                 debug('del: hosted zone not found.', error);
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
             } else if (error && error.code === 'PriorRequestNotComplete') {
                 debug('del: resource is still busy', error);
-                return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
             } else if (error && error.code === 'InvalidChangeBatch') {
                 debug('del: invalid change batch. No such record to be deleted.');
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
             } else if (error) {
                 debug('del: error', error);
-                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
             }
 
             callback(null);
@@ -206,6 +207,7 @@ function getChangeStatus(dnsConfig, changeId, callback) {
 
     var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
     route53.getChange({ Id: changeId }, function (error, result) {
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
         if (error) return callback(error);
 
         callback(null, result.ChangeInfo.Status);

src/docker.js

@@ -1,16 +1,5 @@
 'use strict';
 
-var addons = require('./addons.js'),
-    async = require('async'),
-    assert = require('assert'),
-    config = require('./config.js'),
-    debug = require('debug')('box:src/docker.js'),
-    Docker = require('dockerode'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    util = require('util'),
-    _ = require('underscore');
-
 exports = module.exports = {
     connection: connectionInstance(),
     downloadImage: downloadImage,
@@ -23,10 +12,13 @@ exports = module.exports = {
     deleteContainerByName: deleteContainer,
     deleteImage: deleteImage,
     deleteContainers: deleteContainers,
-    createSubcontainer: createSubcontainer
+    createSubcontainer: createSubcontainer,
+    getContainerIdByIp: getContainerIdByIp,
+    execContainer: execContainer
 };
 
 function connectionInstance() {
+    var Docker = require('dockerode');
     var docker;
 
     if (process.env.BOX_ENV === 'test') {
@@ -42,6 +34,20 @@ function connectionInstance() {
     return docker;
 }
 
+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    child_process = require('child_process'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/docker.js'),
+    once = require('once'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    spawn = child_process.spawn,
+    util = require('util'),
+    _ = require('underscore');
+
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
 
@@ -132,12 +138,13 @@ function createSubcontainer(app, name, cmd, options, callback) {
     var manifest = app.manifest;
     var developmentMode = !!manifest.developmentMode;
     var exposedPorts = {}, dockerPortBindings = { };
+    var domain = app.altDomain || config.appFqdn(app.location);
     var stdEnv = [
         'CLOUDRON=1',
         'WEBADMIN_ORIGIN=' + config.adminOrigin(),
         'API_ORIGIN=' + config.adminOrigin(),
-        'APP_ORIGIN=https://' + config.appFqdn(app.location),
-        'APP_DOMAIN=' + config.appFqdn(app.location)
+        'APP_ORIGIN=https://' + domain,
+        'APP_DOMAIN=' + domain
     ];
 
     // docker portBindings requires ports to be exposed
@@ -156,7 +163,15 @@ function createSubcontainer(app, name, cmd, options, callback) {
         dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
     }
 
-    var memoryLimit = manifest.memoryLimit || (developmentMode ? 0 : 1024 * 1024 * 200); // 200mb by default
+    // first check db record, then manifest
+    var memoryLimit = app.memoryLimit || manifest.memoryLimit;
+
+    // ensure we never go below minimum
+    memoryLimit = memoryLimit < constants.DEFAULT_MEMORY_LIMIT ? constants.DEFAULT_MEMORY_LIMIT : memoryLimit; // 256mb by default
+
+    // developerMode does not restrict memory usage
+    memoryLimit = developmentMode ? 0 : memoryLimit;
+
     // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
     // this means cloudron exec does not work
     var isolatedNetworkNs = true;
@@ -346,3 +361,67 @@ function deleteImage(manifest, callback) {
         callback(error);
     });
 }
+
+function getContainerIdByIp(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('get container by ip %s', ip);
+
+    var docker = exports.connection;
+
+    docker.listNetworks({}, function (error, result) {
+        if (error) return callback(error);
+
+        var bridge;
+        result.forEach(function (n) {
+            if (n.Name === 'bridge') bridge = n;
+        });
+
+        if (!bridge) return callback(new Error('Unable to find the bridge network'));
+
+        var containerId;
+        for (var id in bridge.Containers) {
+            if (bridge.Containers[id].IPv4Address.indexOf(ip) === 0) {
+                containerId = id;
+                break;
+            }
+        }
+        if (!containerId) return callback(new Error('No container with that ip'));
+
+        debug('found container %s with ip %s', containerId, ip);
+
+        callback(null, containerId);
+    });
+}
+
+function execContainer(containerId, cmd, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert(util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', '-i', containerId  ].concat(cmd));
+
+    var chunks = [ ];
+
+    if (options.stdout) {
+        cp.stdout.pipe(options.stdout);
+    } else if (options.bufferStdout) {
+        cp.stdout.on('data', function (chunk) { chunks.push(chunk); });
+    } else {
+        cp.stdout.pipe(process.stdout);
+    }
+
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debug('execContainer code: %s signal: %s', code, signal);
+        if (!callback.called) callback(code ? 'Failed with status ' + code : null, Buffer.concat(chunks));
+    });
+
+    cp.stderr.pipe(options.stderr || process.stderr);
+
+    if (options.stdin) options.stdin.pipe(cp.stdin).on('error', callback);
+}

src/eventlog.js

@@ -0,0 +1,101 @@
+'use strict';
+
+exports = module.exports = {
+    EventLogError: EventLogError,
+
+    add: add,
+    get: get,
+    getAllPaged: getAllPaged,
+
+    // keep in sync with webadmin index.js filter
+    ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CONFIGURE: 'app.configure',
+    ACTION_APP_INSTALL: 'app.install',
+    ACTION_APP_RESTORE: 'app.restore',
+    ACTION_APP_UNINSTALL: 'app.uninstall',
+    ACTION_APP_UPDATE: 'app.update',
+    ACTION_BACKUP_FINISH: 'backup.finish',
+    ACTION_BACKUP_START: 'backup.start',
+    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CLI_MODE: 'settings.climode',
+    ACTION_START: 'cloudron.start',
+    ACTION_UPDATE: 'cloudron.update',
+    ACTION_USER_ADD: 'user.add',
+    ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_REMOVE: 'user.remove',
+    ACTION_USER_UPDATE: 'user.update'
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:eventlog'),
+    eventlogdb = require('./eventlogdb.js'),
+    util = require('util'),
+    uuid = require('node-uuid');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function EventLogError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(EventLogError, Error);
+EventLogError.INTERNAL_ERROR = 'Internal error';
+EventLogError.NOT_FOUND = 'Not Found';
+
+function add(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    var id = uuid.v4();
+
+    eventlogdb.add(id, action, source, data, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, { id: id });
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new EventLogError(EventLogError.NOT_FOUND, 'No such event'));
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.getAllPaged(action, search, page, perPage, function (error, boxes) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, boxes);
+    });
+}

src/eventlogdb.js

@@ -0,0 +1,104 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getAllPaged: getAllPaged,
+    add: add,
+    count: count,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    mysql = require('mysql'),
+    safe = require('safetydance');
+
+var EVENTLOGS_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+
+// until mysql module supports automatic type coercion
+function postProcess(eventLog) {
+    eventLog.source = safe.JSON.parse(eventLog.source);
+    eventLog.data = safe.JSON.parse(eventLog.data);
+    return eventLog;
+}
+
+function get(eventId, callback) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE id = ?', [ eventId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, postProcess(result[0]));
+    });
+}
+
+function getAllPaged(action, search, page, perPage, callback) {
+    assert(typeof action === 'string' || action === null);
+    assert(typeof search === 'string' || search === null);
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [];
+    var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog';
+
+    if (action || search) query += ' WHERE';
+    if (search) query += ' data LIKE ' + mysql.escape('%' + search + '%');
+    if (action && search) query += ' AND ';
+
+    if (action) {
+        query += ' action=?';
+        data.push(action);
+    }
+
+    query += ' ORDER BY creationTime DESC LIMIT ?,?';
+
+    data.push((page-1)*perPage);
+    data.push(perPage);
+
+    database.query(query, data, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, action, source, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM eventlog', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM eventlog', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
+

src/groupdb.js

@@ -0,0 +1,216 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
+    add: add,
+    del: del,
+    count: count,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror');
+
+var GROUPS_FIELDS = [ 'id', 'name' ].join(',');
+
+function get(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups WHERE id = ?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result[0]);
+    });
+}
+
+function getWithMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' WHERE groups.id = ? ' +
+                    ' GROUP BY groups.id', [ groupId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        var result = results[0];
+        result.userIds = result.userIds ? result.userIds.split(',') : [ ];
+
+        callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' GROUP BY groups.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
+function add(id, name, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [ id, name ];
+    database.query('INSERT INTO groups (id, name) VALUES (?, ?)',
+           data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
+    queries.push({ query: 'DELETE FROM groups WHERE id = ?', args: [ id ] });
+
+    database.transaction(queries, function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(error);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM groupMembers', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        database.query('DELETE FROM groups WHERE id != ?', [ 'admin' ], function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            callback(error);
+        });
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.userId; }));
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.groupId; }));
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [ ];
+    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
+    groupIds.forEach(function (gid) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result.length !== 0);
+    });
+}

src/groups.js

@@ -0,0 +1,221 @@
+/* jshint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    GroupError: GroupError,
+
+    create: create,
+    remove: remove,
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    ADMIN_GROUP_ID: 'admin' // see db migration code and groupdb._clear
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    groupdb = require('./groupdb.js'),
+    util = require('util');
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function GroupError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(GroupError, Error);
+GroupError.INTERNAL_ERROR = 'Internal Error';
+GroupError.ALREADY_EXISTS = 'Already Exists';
+GroupError.NOT_FOUND = 'Not Found';
+GroupError.BAD_FIELD = 'Field error';
+GroupError.NOT_EMPTY = 'Not Empty';
+GroupError.NOT_ALLOWED = 'Not Allowed';
+
+function validateGroupname(name) {
+    assert.strictEqual(typeof name, 'string');
+    var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
+
+    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');
+
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');
+
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
+
+    return null;
+}
+
+function create(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateGroupname(name);
+    if (error) return callback(error);
+
+    groupdb.add(name /* id */, name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new GroupError(GroupError.ALREADY_EXISTS));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null, { id: name, name: name });
+    });
+}
+
+function remove(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // never allow admin group to be deleted
+    if (id === exports.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
+
+    groupdb.del(id, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getWithMembers(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getWithMembers(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAll(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getMembers(groupId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getGroups(userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.setGroups(userId, groupIds, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.addMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.removeMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.isMember(groupId, userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}

src/infra_version.js

@@ -0,0 +1,21 @@
+'use strict';
+
+// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+// These constants are used in the installer script as well
+// Do not require anything here!
+
+exports = module.exports = {
+    'version': 36,
+
+    'baseImage': 'cloudron/base:0.8.1',
+
+    'images': {
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.11.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.10.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.9.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.8.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.13.2' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.8.0' }
+    }
+};
+

src/ldap.js

@@ -6,11 +6,15 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    apps = require('./apps.js'),
     config = require('./config.js'),
     debug = require('debug')('box:ldap'),
+    eventlog = require('./eventlog.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    ldap = require('ldapjs');
+    ldap = require('ldapjs'),
+    mailboxes = require('./mailboxes.js'),
+    MailboxError = mailboxes.MailboxError;
 
 var gServer = null;
 
@@ -28,104 +32,237 @@ var gLogger = {
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
 var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
+function getAppByRequest(req, callback) {
+    var sourceIp = req.connection.ldap.id.split(':')[0];
+    if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
+
+    apps.getByIpAddress(sourceIp, function (error, app) {
+        // we currently allow access in case we can't find the source app
+        callback(null, app || null);
+    });
+}
+
+function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // send user objects
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+            var groups = [ GROUP_USERS_DN ];
+            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+            var displayName = entry.displayName || entry.username;
+            var nameParts = displayName.split(' ');
+            var firstName = nameParts[0];
+            var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['user'],
+                    objectcategory: 'person',
+                    cn: entry.id,
+                    uid: entry.id,
+                    mail: entry.email,
+                    // TODO: check mailboxes before we send this
+                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    displayname: displayName,
+                    givenName: firstName,
+                    username: entry.username,
+                    samaccountname: entry.username,      // to support ActiveDirectory clients
+                    memberof: groups
+                }
+            };
+
+            // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+            // which is required to have atleast one character if present
+            if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result){
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        var groups = [{
+            name: 'users',
+            admin: false
+        }, {
+            name: 'admins',
+            admin: true
+        }];
+
+        groups.forEach(function (group) {
+            var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: group.name,
+                    memberuid: members.map(function(entry) { return entry.id; })
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function mailboxSearch(req, res, next) {
+    debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
+
+            // TODO: send aliases
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['mailbox'],
+                    objectcategory: 'mailbox',
+                    cn: entry.name,
+                    uid: entry.name,
+                    mail: entry.name + '@' + config.fqdn()
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function authenticateUser(req, res, next) {
+    debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+    // extract the common name which might have different attribute names
+    var attributeName = Object.keys(req.dn.rdns[0])[0];
+    var commonName = req.dn.rdns[0][attributeName];
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    var api;
+    if (attributeName === 'mail') {
+        api = user.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        var parts = commonName.split('@');
+        if (parts[1] === config.fqdn()) { // internal email, verify with username
+            commonName = parts[0];
+            api = user.verifyWithUsername;
+        } else { // external email
+            api = user.verifyWithEmail;
+        }
+    } else if (commonName.indexOf('uid-') === 0) {
+        api = user.verify;
+    } else {
+        api = user.verifyWithUsername;
+    }
+
+    api(commonName, req.credentials || '', function (error, user) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorizeUserForApp(req, res, next) {
+    assert(req.user);
+
+    // We simply authorize the user to access a mailbox by his own name
+    getAppByRequest(req, function (error, app) {
+        if (error) return next(error);
+
+        if (!app) {
+            debug('no app found for this container, allow access');
+            return res.end();
+        }
+
+        apps.hasAccessTo(app, req.user, function (error, result) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we return no such object, to avoid leakage of a users existence
+            if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: req.user.id });
+
+            res.end();
+        });
+    });
+}
+
+function authorizeUserForMailbox(req, res, next) {
+    assert(req.user);
+
+    mailboxes.get(req.user.username, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+
+        res.end();
+    });
+}
+
 function start(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     gServer = ldap.createServer({ log: gLogger });
 
-    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+    gServer.search('ou=users,dc=cloudron', userSearch);
+    gServer.search('ou=groups,dc=cloudron', groupSearch);
+    gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);
 
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);
 
-            // send user objects
-            result.forEach(function (entry) {
-                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
-
-                var groups = [ GROUP_USERS_DN ];
-                if (entry.admin) groups.push(GROUP_ADMINS_DN);
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['user'],
-                        objectcategory: 'person',
-                        cn: entry.id,
-                        uid: entry.id,
-                        mail: entry.email,
-                        displayname: entry.displayName || entry.username,
-                        username: entry.username,
-                        samaccountname: entry.username,      // to support ActiveDirectory clients
-                        memberof: groups
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
-
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
-
-            var groups = [{
-                name: 'users',
-                admin: false
-            }, {
-                name: 'admins',
-                admin: true
-            }];
-
-            groups.forEach(function (group) {
-                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
-
-                var tmp = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['group'],
-                        cn: group.name,
-                        memberuid: members.map(function(entry) { return entry.id; })
-                    }
-                };
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
-        // TODO: validate password
-        debug('ldap application bind: %s', req.dn.toString());
+    // this is the bind for addons (after bind, they might search and authenticate)
+    gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
+        debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
         res.end();
     });
 
-    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('ldap user bind: %s', req.dn.toString());
-
-        // extract the common name which might have different attribute names
-        var commonName = req.dn.rdns[0][Object.keys(req.dn.rdns[0])[0]];
-        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-        user.verify(commonName, req.credentials || '', function (error, result) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-            if (error) return next(new ldap.OperationsError(error));
-
-            res.end();
-        });
+    // this is the bind for apps (after bind, they might search and authenticate user)
+    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
+        // TODO: validate password
+        debug('application bind: %s', req.dn.toString());
+        res.end();
     });
 
     gServer.listen(config.get('ldapPort'), callback);

src/logcollector.js

@@ -0,0 +1,37 @@
+'use strict';
+
+exports = module.exports = {
+    sendFailureLogs: sendFailureLogs
+};
+
+var assert = require('assert'),
+    mailer = require('./mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
+
+function collectLogs(unitName, callback) {
+    assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendFailureLogs(processName, options) {
+    assert.strictEqual(typeof processName, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    collectLogs(options.unit || processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending failure logs for', processName);
+
+        mailer.unexpectedExit(processName, result);
+    });
+}

src/mail_templates/app_update_available.ejs

@@ -2,10 +2,13 @@
 
 Dear Admin,
 
-A new version of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+A new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
 
 The app will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
 
+Changes:
+<%= updateInfo.manifest.changelog %>
+
 Thank you,
 your Cloudron
 

src/mail_templates/box_update_available.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-A new version of Cloudron <%= fqdn %> is available!
+Version <%= newBoxVersion %> of Cloudron <%= fqdn %> is now available!
 
 Your Cloudron will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
 

src/mail_templates/certificate_renewed.ejs

@@ -0,0 +1,16 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+<% if (message) { %>
+<%= domain %> was not renewed.
+
+<%- message %>
+<% } else { %>
+<%= domain %> was renewed.
+<% } %>
+Thank you,
+Your Cloudron
+<% } else { %>
+
+<% } %>
+

src/mail_templates/password_reset.ejs

@@ -1,6 +1,6 @@
 <%if (format === 'text') { %>
 
-Dear <%= username %>,
+Dear <%= user.username || user.email %>,
 
 Someone, hopefully you, has requested your <%= fqdn %>'s account password
 be reset. If you did not request this reset, please ignore this message.

src/mail_templates/unexpected_exit.ejs

@@ -2,7 +2,7 @@
 
 Dear Cloudron Team,
 
-Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> exited unexpectedly!
 
 Please see some excerpt of the logs below.
 

src/mail_templates/user_added.ejs

@@ -2,16 +2,16 @@
 
 Dear Admin,
 
-User with name '<%= username %>' (<%= email %>) was added in the Cloudron at <%= fqdn %>.
+User with email <%= user.email %> was added in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 
 <% if (inviteLink) { %>
-This user was not invited immediately, he has to get invited manually later, using the "send invite" button in the admin panel.
-To perform any configuration on behalf of the user, please use this link
+As requested, this user has not been sent an invitation email.
+
+To set a password and perform any configuration on behalf of the user, please use this link:
 <%= inviteLink %>
-It allows to setup a temporary password, which the user will be able to override, once he gets invited.
-This link will become invalid as soon as the user was invited.
+
 <% } %>
 
 Thank you,

src/mail_templates/user_event.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-User with name '<%= username %>' (<%= email %>) <%= event %> in the Cloudron at <%= fqdn %>.
+User <%= user.username %> <%= user.email %> <%= event %> in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mail_templates/welcome_user.ejs

@@ -1,20 +1,15 @@
 <%if (format === 'text') { %>
 
-Dear <%= user.username %>,
+Dear <%= user.email %>,
 
-Welcome to my Cloudron <%= fqdn %>!
+Welcome to our Cloudron <%= fqdn %>!
 
 The Cloudron is our own Smart Server. You can read more about it
 at https://www.cloudron.io.
 
-           You username is '<%= user.username %>'
-
 To get started, create your account by visiting the following page:
 <%= setupLink %>
 
-When you visit the above page, you will be prompted to enter a new password.
-After you have submitted the form, you can login using the new password.
-
 <% if (invitor && invitor.email) { %>
 Thank you,
 <%= invitor.email %>

src/mailboxdb.js

@@ -0,0 +1,127 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    getAliases: getAliases,
+    setAliases: setAliases,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'aliasTarget', 'creationTime' ].join(',');
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name) VALUES (?)', [ name ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE name=? OR aliasTarget = ?', [ name, name ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function postProcess(result) {
+    result.aliases = result.aliases ? result.aliases.split(',') : [ ];
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.name=? AND m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(results[0]);
+
+        callback(null, results[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ?', args: [ name ] });
+    aliases.forEach(function (alias) {
+        queries.push({ query: 'INSERT INTO mailboxes (name, aliasTarget) VALUES (?, ?)', args: [ alias, name ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name FROM mailboxes WHERE aliasTarget=? ORDER BY name', [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results = results.map(function (r) { return r.name; });
+        callback(null, results);
+    });
+}

src/mailboxes.js

@@ -0,0 +1,198 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    setAliases: setAliases,
+    getAliases: getAliases,
+
+    setupAliases: setupAliases,
+
+    MailboxError: MailboxError
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:mailboxes'),
+    docker = require('./docker.js'),
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util');
+
+function MailboxError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(MailboxError, Error);
+MailboxError.ALREADY_EXISTS = 'already exists';
+MailboxError.BAD_FIELD = 'Field error';
+MailboxError.NOT_FOUND = 'not found';
+MailboxError.INTERNAL_ERROR = 'internal error';
+MailboxError.EXTERNAL_ERROR = 'external error';
+
+function validateName(name) {
+    var RESERVED_NAMES = [ 'no-reply', 'postmaster', 'mailer-daemon' ];
+
+    if (!name.length) return new MailboxError(MailboxError.BAD_FIELD, "name cannot be empty");
+
+    if (name.length < 2) return new MailboxError(MailboxError.BAD_FIELD, 'name too small');
+    if (name.length > 127) return new MailboxError(MailboxError.BAD_FIELD, 'name too long');
+    if (RESERVED_NAMES.indexOf(name) !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'name is reserved');
+
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailboxError(MailboxError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    if (name.indexOf('.app') !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'alias pattern is reserved for apps');
+
+    return null;
+}
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.add(name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        debug('Added mailbox %s', name);
+
+        var mailbox = {
+            name: name
+        };
+
+        callback(null, mailbox);
+    });
+}
+
+function pushAlias(name, aliases, callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    var cmd = [ '/addons/mail/service.sh', 'set-alias', name ].concat(aliases);
+
+    debug('pushing alias for %s : %j', name, aliases);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(new MailboxError(MailboxError.EXTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    pushAlias(name, [ ], function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.del(name, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+            debug('deleted mailbox %s', name);
+
+            callback();
+        });
+    });
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.get(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, mailbox);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAll(function (error, results) {
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var i = 0; i < aliases.length; i++) {
+        aliases[i] = aliases[i].toLowerCase();
+
+        var error = validateName(aliases[i]);
+        if (error) return callback(error);
+    }
+
+    pushAlias(name, aliases, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.setAliases(name, aliases, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS, error.message))
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+
+            callback(null);
+        });
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAliases(name, function (error, aliases) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, aliases);
+    });
+}
+
+// push aliases to the mail container on startup
+function setupAliases(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, mailboxes) {
+        if (error) return callback(error);
+
+        async.each(mailboxes, function (mailbox, iteratorDone) {
+            getAliases(mailbox.name, function (error, aliases) {
+                if (error) return iteratorDone(error);
+
+                if (aliases.length === 0) return iteratorDone();
+
+                pushAlias(mailbox.name, aliases, iteratorDone);
+            });
+        }, callback)
+    });
+}
+

src/mailer.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -14,15 +12,19 @@ exports = module.exports = {
     appUpdateAvailable: appUpdateAvailable,
 
     sendInvite: sendInvite,
-    sendCrashNotification: sendCrashNotification,
+    unexpectedExit: unexpectedExit,
 
     appDied: appDied,
 
     outOfDiskSpace: outOfDiskSpace,
 
+    certificateRenewed: certificateRenewed,
+
     FEEDBACK_TYPE_FEEDBACK: 'feedback',
     FEEDBACK_TYPE_TICKET: 'ticket',
-    FEEDBACK_TYPE_APP: 'app',
+    FEEDBACK_TYPE_APP_MISSING: 'app_missing',
+    FEEDBACK_TYPE_APP_ERROR: 'app_error',
+    FEEDBACK_TYPE_UPGRADE_REQUEST: 'upgrade_request',
     sendFeedback: sendFeedback,
 
     _getMailQueue: _getMailQueue,
@@ -39,6 +41,7 @@ var assert = require('assert'),
     ejs = require('ejs'),
     nodemailer = require('nodemailer'),
     path = require('path'),
+    platform = require('./platform.js'),
     safe = require('safetydance'),
     smtpTransport = require('nodemailer-smtp-transport'),
     users = require('./user.js'),
@@ -107,6 +110,7 @@ function getTxtRecords(callback) {
     });
 }
 
+// keep this in sync with the cloudron.js dns changes
 function checkDns() {
     getTxtRecords(function (error, records) {
         if (error || !records) {
@@ -120,7 +124,7 @@ function checkDns() {
         for (var i = 0; i < records.length; i++) {
             if (records[i].indexOf('v=spf1 ') !== 0) continue; // not SPF
 
-            allowedToSendMail = records[i].indexOf('a:' + config.fqdn()) !== -1;
+            allowedToSendMail = records[i].indexOf('a:' + config.adminFqdn()) !== -1;
             break; // only one SPF record can exist (https://support.google.com/a/answer/4568483?hl=en)
         }
 
@@ -156,10 +160,14 @@ function sendMails(queue) {
 
         var transport = nodemailer.createTransport(smtpTransport({
             host: mailServerIp,
-            port: 2500 // this value comes from mail container
+            port: config.get('smtpPort'),
+            auth: {
+                user: platform.mailConfig().username,
+                pass: platform.mailConfig().password
+            }
         }));
 
-        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
+        debug('Processing mail queue of size %d (through %s:2525)', queue.length, mailServerIp);
 
         async.mapSeries(queue, function iterator(mailOptions, callback) {
             transport.sendMail(mailOptions, function (error) {
@@ -176,8 +184,8 @@ function sendMails(queue) {
 function enqueue(mailOptions) {
     assert.strictEqual(typeof mailOptions, 'object');
 
-    if (!mailOptions.from) console.error('from is missing');
-    if (!mailOptions.to) console.error('to is missing');
+    if (!mailOptions.from) console.error('sender address is missing');
+    if (!mailOptions.to) console.error('recipient address is missing');
 
     debug('Queued mail for ' + mailOptions.from + ' to ' + mailOptions.to);
     gMailQueue.push(mailOptions);
@@ -196,6 +204,8 @@ function getAdminEmails(callback) {
     users.getAllAdmins(function (error, admins) {
         if (error) return callback(error);
 
+        if (admins.length === 0) return callback(new Error('No admins on this cloudron')); // box not activated yet
+
         var adminEmails = [ ];
         admins.forEach(function (admin) { adminEmails.push(admin.email); });
 
@@ -213,10 +223,10 @@ function mailUserEventToAdmins(user, event) {
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
-            subject: util.format('%s %s in Cloudron %s', user.username, event, config.fqdn()),
-            text: render('user_event.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, event: event, format: 'text' }),
+            subject: util.format('%s %s in Cloudron %s', user.username || user.email, event, config.fqdn()),
+            text: render('user_event.ejs', { fqdn: config.fqdn(), user: user, event: event, format: 'text' }),
         };
 
         enqueue(mailOptions);
@@ -232,14 +242,14 @@ function sendInvite(user, invitor) {
     var templateData = {
         user: user,
         webadminUrl: config.adminOrigin(),
-        setupLink: config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken,
+        setupLink: config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken,
         format: 'text',
         fqdn: config.fqdn(),
         invitor: invitor
     };
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: util.format('Welcome to Cloudron %s', config.fqdn()),
         text: render('welcome_user.ejs', templateData)
@@ -259,47 +269,48 @@ function userAdded(user, inviteSent) {
 
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
-        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken;
+        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken;
 
         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
-            subject: util.format('%s added in Cloudron %s', user.username, config.fqdn()),
-            text: render('user_added.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, inviteLink: inviteLink, format: 'text' }),
+            subject: util.format('%s added in Cloudron %s', user.email, config.fqdn()),
+            text: render('user_added.ejs', { fqdn: config.fqdn(), user: user, inviteLink: inviteLink, format: 'text' }),
         };
 
         enqueue(mailOptions);
     });
 }
 
-function userRemoved(username) {
-    assert.strictEqual(typeof username, 'string');
+function userRemoved(user) {
+    assert.strictEqual(typeof user, 'object');
 
-    debug('Sending mail for userRemoved');
+    debug('Sending mail for userRemoved.', user.id, user.email);
 
-    mailUserEventToAdmins({ username: username }, 'was removed');
+    mailUserEventToAdmins(user, 'was removed');
 }
 
-function adminChanged(user) {
+function adminChanged(user, admin) {
     assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof admin, 'boolean');
 
     debug('Sending mail for adminChanged');
 
-    mailUserEventToAdmins(user, user.admin ? 'is now an admin' : 'is no more an admin');
+    mailUserEventToAdmins(user, admin ? 'is now an admin' : 'is no more an admin');
 }
 
 function passwordReset(user) {
     assert.strictEqual(typeof user, 'object');
 
-    debug('Sending mail for password reset for user %s.', user.username);
+    debug('Sending mail for password reset for user %s.', user.email, user.id);
 
     var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: 'Password Reset Request',
-        text: render('password_reset.ejs', { fqdn: config.fqdn(), username: user.username, resetLink: resetLink, format: 'text' })
+        text: render('password_reset.ejs', { fqdn: config.fqdn(), user: user, resetLink: resetLink, format: 'text' })
     };
 
     enqueue(mailOptions);
@@ -314,7 +325,7 @@ function appDied(app) {
         if (error) return console.log('Error getting admins', error);
 
         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
             to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
@@ -332,7 +343,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', config.fqdn()),
             text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
@@ -350,10 +361,10 @@ function appUpdateAvailable(app, updateInfo) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.get('adminEmail'),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', app.fqdn),
-            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, format: 'text' })
+            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
         };
 
         enqueue(mailOptions);
@@ -364,7 +375,7 @@ function outOfDiskSpace(message) {
     assert.strictEqual(typeof message, 'string');
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] Out of disk space alert', config.fqdn()),
         text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
@@ -373,17 +384,31 @@ function outOfDiskSpace(message) {
     sendMails([ mailOptions ]);
 }
 
+function certificateRenewed(domain, message) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof message, 'string');
+
+    var mailOptions = {
+        from: platform.mailConfig().from,
+        to: 'admin@cloudron.io',
+        subject: util.format('[%s] Certificate was %s renewed', domain, message ? 'not' : ''),
+        text: render('certificate_renewed.ejs', { domain: domain, message: message, format: 'text' })
+    };
+
+    sendMails([ mailOptions ]);
+}
+
 // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
 // crashnotifier should be able to send mail when there is no db
-function sendCrashNotification(program, context) {
+function unexpectedExit(program, context) {
     assert.strictEqual(typeof program, 'string');
     assert.strictEqual(typeof context, 'string');
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
-        text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
+        text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
     };
 
     sendMails([ mailOptions ]);
@@ -395,10 +420,14 @@ function sendFeedback(user, type, subject, description) {
     assert.strictEqual(typeof subject, 'string');
     assert.strictEqual(typeof description, 'string');
 
-    assert(type === exports.FEEDBACK_TYPE_TICKET || type === exports.FEEDBACK_TYPE_FEEDBACK || type === exports.FEEDBACK_TYPE_APP);
+    assert(type === exports.FEEDBACK_TYPE_TICKET ||
+        type === exports.FEEDBACK_TYPE_FEEDBACK ||
+        type === exports.FEEDBACK_TYPE_APP_MISSING ||
+        type === exports.FEEDBACK_TYPE_UPGRADE_REQUEST ||
+        type === exports.FEEDBACK_TYPE_APP_ERROR);
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: platform.mailConfig().from,
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})
@@ -411,6 +440,8 @@ function _getMailQueue() {
     return gMailQueue;
 }
 
-function _clearMailQueue() {
+function _clearMailQueue(callback) {
     gMailQueue = [];
+
+    if (callback) callback();
 }

src/middleware/index.js

@@ -4,7 +4,6 @@ exports = module.exports = {
     cookieParser: require('cookie-parser'),
     cors: require('./cors'),
     csrf: require('csurf'),
-    favicon: require('serve-favicon'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),

src/nginx.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var assert = require('assert'),
@@ -13,6 +11,7 @@ var assert = require('assert'),
     shell = require('./shell.js');
 
 exports = module.exports = {
+    requiresOAuthProxy: requiresOAuthProxy,
     configureAdmin: configureAdmin,
     configureApp: configureApp,
     unconfigureApp: unconfigureApp,
@@ -22,6 +21,19 @@ exports = module.exports = {
 var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
     RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
 
+function requiresOAuthProxy(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    var tmp = app.accessRestriction;
+
+    // if no accessRestriction set, or the app uses one of the auth modules, we do not need the oauth proxy
+    if (tmp === null) return false;
+    if (app.manifest.addons['ldap'] || app.manifest.addons['oauth'] || app.manifest.addons['simpleauth']) return false;
+
+    // check if any restrictions are set
+    return !!((tmp.users && tmp.users.length) || (tmp.groups && tmp.groups.length));
+}
+
 function configureAdmin(certFilePath, keyFilePath, callback) {
     assert.strictEqual(typeof certFilePath, 'string');
     assert.strictEqual(typeof keyFilePath, 'string');
@@ -50,8 +62,9 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var sourceDir = path.resolve(__dirname, '..');
-    var endpoint = app.oauthProxy ? 'oauthproxy' : 'app';
-    var vhost = config.appFqdn(app.location);
+    var oauthProxy = requiresOAuthProxy(app);
+    var endpoint = oauthProxy ? 'oauthproxy' : 'app';
+    var vhost = app.altDomain || config.appFqdn(app.location);
 
     var data = {
         sourceDir: sourceDir,
@@ -65,10 +78,10 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writing config for "%s" to %s', app.location, nginxConfigFilename);
+    debug('writing config for "%s" to %s', vhost, nginxConfigFilename);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-        debug('Error creating nginx config for "%s" : %s', app.location, safe.error.message);
+        debug('Error creating nginx config for "%s" : %s', vhost, safe.error.message);
         return callback(safe.error);
     }
 
@@ -79,9 +92,11 @@ function unconfigureApp(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    var vhost = app.altDomain || config.appFqdn(app.location);
+
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
     if (!safe.fs.unlinkSync(nginxConfigFilename)) {
-        debug('Error removing nginx configuration of "%s": %s', app.location, safe.error.message);
+        debug('Error removing nginx configuration of "%s": %s', vhost, safe.error.message);
         return callback(null);
     }
 

src/oauth2views/account_setup.ejs

@@ -0,0 +1,78 @@
+<% include header %>
+
+<!-- tester -->
+
+<script>
+
+'use strict';
+
+// very basic angular app
+var app = angular.module('Application', []);
+app.controller('Controller', ['$scope', function ($scope) {
+    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
+    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
+}]);
+
+</script>
+
+<center>
+    <br/>
+    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to your Cloudron.</h4>
+    <h2>Setup your account and password.</h2>
+</center>
+
+<div class="container" ng-app="Application" ng-controller="Controller">
+    <div class="row">
+        <div class="col-md-6 col-md-offset-3">
+            <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
+                <input type="password" style="display: none;">
+                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
+
+                <center><p class="has-error"><%= error %></p></center>
+
+<% if (user && user.username) { %>
+                <div class="form-group"">
+                    <label class="control-label">Username</label>
+                    <input type="text" class="form-control" ng-model="username" name="username" readonly required>
+                </div>
+<% } else { %>
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
+                    <label class="control-label">Username</label>
+                    <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
+                        <small ng-show="setupForm.username.$error.minlength">The username is too short</small>
+                        <small ng-show="setupForm.username.$error.maxlength">The username is too long</small>
+                        <small ng-show="setupForm.username.$dirty && setupForm.username.$invalid">Not a valid username</small>
+                    </div>
+                    <input type="text" class="form-control" ng-model="username" name="username" ng-maxlength="512" ng-minlength="3" required autofocus>
+                </div>
+<% } %>
+
+                <div class="form-group">
+                    <label class="control-label">Display Name</label>
+                    <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
+                </div>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
+                    <label class="control-label">New Password</label>
+                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
+                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+                    </div>
+                    <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" required>
+                </div>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
+                    <label class="control-label">Repeat Password</label>
+                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
+                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
+                    </div>
+                    <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
+                </div>
+
+                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
+            </form>
+        </div>
+    </div>
+</div>
+
+<% include footer %>

src/oauth2views/header.ejs

@@ -8,9 +8,11 @@
 
     <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
 
+    <!-- Theme CSS -->
+    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
-    <link href="https://fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
     <!-- jQuery-->
     <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
@@ -22,9 +24,6 @@
     <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
     <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
-
 </head>
 
 <body class="oauth">

src/oauth2views/password_reset.ejs

@@ -13,13 +13,14 @@ app.controller('Controller', [function () {}]);
 </script>
 
 <center>
-    <h1>Hello <%= user.username %> create a new password</h1>
+    <h1>Hello <%= user.username %>, set a new password</h1>
 </center>
 
 <div class="container" ng-app="Application" ng-controller="Controller">
     <div class="row">
         <div class="col-md-6 col-md-offset-3">
             <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
+                <input type="password" style="display: none;">
                 <input type="hidden" name="_csrf" value="<%= csrf %>"/>
                 <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
 

src/oauth2views/password_reset_sent.ejs

@@ -3,14 +3,14 @@
 <!-- tester -->
 
 <center>
-    <h1>Reset your password successful</h1>
+    <h1>Password reset successful</h1>
 </center>
 
 <div class="container">
     <div class="row">
         <div class="col-md-6 col-md-offset-3">
-            <p>An email was sent to you with a link to create a new password.</p>
-            If you have not received any email after some time, maybe you have misspelled your email address, simply try again <a href="/api/v1/session/password/resetRequest.html">here</a>.
+            <p>An email was sent to you with a link to set a new password.</p>
+            If you have not received any email, simply <a href="/api/v1/session/password/resetRequest.html">try again</a>.
         </div>
     </div>
 </div>

src/oauth2views/password_setup.ejs

@@ -1,46 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
-
-</script>
-
-<center>
-    <h1>Hello <%= user.username %> create a password</h1>
-</center>
-
-<div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-        <div class="col-md-6 col-md-offset-3">
-            <form action="/api/v1/session/password/reset" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
-                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-                <div class="form-group" ng-class="{ 'has-error': setupForm.password.$dirty && setupForm.password.$invalid }">
-                    <label class="control-label" for="inputPassword">New Password</label>
-                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
-                    </div>
-                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" autofocus required>
-                </div>
-                <div class="form-group" ng-class="{ 'has-error': setupForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
-                    <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
-                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-                    </div>
-                    <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
-                </div>
-                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
-            </form>
-        </div>
-    </div>
-</div>
-
-<% include footer %>

src/oauthproxy.js

@@ -7,7 +7,7 @@ exports = module.exports = {
 
 var appdb = require('./appdb.js'),
     assert = require('assert'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:proxy'),
@@ -124,9 +124,9 @@ function authenticate(req, res, next) {
                 return res.send(500, 'Unknown app.');
             }
 
-            clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
+            clients.getByAppIdAndType(result.id, clients.TYPE_PROXY, function (error, result) {
                 if (error) {
-                    console.error('Unkonwn OAuth client.', error);
+                    console.error('Unknown OAuth client.', error);
                     return res.send(500, 'Unknown OAuth client.');
                 }
 

src/paths.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var config = require('./config.js'),
@@ -13,8 +11,6 @@ exports = module.exports = {
 
     ADDON_CONFIG_DIR: path.join(config.baseDir(), 'data/addons'),
 
-    DNS_IN_SYNC_FILE: path.join(config.baseDir(), 'data/dns_in_sync'),
-
     COLLECTD_APPCONFIG_DIR: path.join(config.baseDir(), 'data/collectd/collectd.conf.d'),
 
     DATA_DIR: path.join(config.baseDir(), 'data'),
@@ -26,9 +22,12 @@ exports = module.exports = {
 
     CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'data/box/avatar.png'),
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
+    FIRST_RUN_FILE: path.join(config.baseDir(), 'data/box/first_run'),
 
     UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
 
     ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
-    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key')
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key'),
+
+    INFRA_VERSION_FILE: path.join(config.baseDir(), 'data/INFRA_VERSION')
 };

src/platform.js

@@ -0,0 +1,108 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+
+    mailConfig: mailConfig
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    certificates = require('./certificates.js'),
+    debug = require('debug')('box:platform'),
+    fs = require('fs'),
+    infra = require('./infra_version.js'),
+    ini = require('ini'),
+    mailboxes = require('./mailboxes.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    util = require('util');
+
+var SETUP_INFRA_CMD = path.join(__dirname, 'scripts/setup_infra.sh');
+
+var gAddonVars = null;
+
+function initialize(callback) {
+    if (process.env.BOX_ENV === 'test' && !process.env.CREATE_INFRA) return callback();
+
+    debug('initializing addon infrastructure');
+
+    var existingInfra = { version: 'none' };
+    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
+        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
+        if (!existingInfra) existingInfra = { version: 'legacy' };
+    }
+
+    if (infra.version === existingInfra.version) {
+        debug('platform is uptodate at version %s', infra.version);
+        return loadAddonVars(callback);
+    }
+
+    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
+
+    async.series([
+        stopContainers,
+        startAddons,
+        removeOldImages,
+        existingInfra.version === 'none' ? apps.restoreInstalledApps : apps.configureInstalledApps,
+        loadAddonVars,
+        mailboxes.setupAliases,
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra))
+    ], callback);
+}
+
+function removeOldImages(callback) {
+    debug('removing old addon images');
+
+    for (var imageName in infra.images) {
+        var image = infra.images[imageName];
+        debug('cleaning up images of %j', image);
+        var cmd = 'docker images "%s" | tail -n +2 | awk \'{ print $1 ":" $2 }\' | grep -v "%s" | xargs --no-run-if-empty docker rmi';
+        shell.execSync('removeOldImagesSync', util.format(cmd, image.repo, image.tag));
+    }
+
+    callback();
+}
+
+function stopContainers(callback) {
+    // TODO: be nice and stop addons cleanly (example, shutdown commands)
+    debug('stopping existing containers');
+    shell.execSync('stopContainersSync', 'docker ps -qa | xargs --no-run-if-empty docker rm -f');
+    callback();
+}
+
+function startAddons(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
+        if (error) return callback(error);
+
+        shell.sudo('setup_infra', [ SETUP_INFRA_CMD, paths.DATA_DIR, config.fqdn(), config.adminFqdn(), certFilePath, keyFilePath ], function (error) {
+            callback(error);
+        });
+    });
+}
+
+function loadAddonVars(callback) {
+    gAddonVars = {
+        mail: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mail_vars.sh', 'utf8')),
+        postgresql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'utf8')),
+        mysql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 'utf8')),
+        mongodb: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'utf8'))
+    };
+    callback();
+}
+
+function mailConfig() {
+    if (!gAddonVars) return { username: 'no-reply', from: 'no-reply@' + config.fqdn(), password: 'doesnotwork' }; // for tests which don't run infra
+
+    return {
+        username: gAddonVars.mail.MAIL_ROOT_USERNAME,
+        from: '"Cloudron" <' + gAddonVars.mail.MAIL_ROOT_USERNAME + '@' + config.fqdn() + '>',
+        password: gAddonVars.mail.MAIL_ROOT_PASSWORD
+    };
+}

src/routes/apps.js

@@ -1,10 +1,7 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
     getApp: getApp,
-    getAppBySubdomain: getAppBySubdomain,
     getApps: getApps,
     getAppIcon: getAppIcon,
     installApp: installApp,
@@ -31,8 +28,12 @@ var apps = require('../apps.js'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     paths = require('../paths.js'),
     safe = require('safetydance'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
 
 function removeInternalAppFields(app) {
     return {
@@ -44,12 +45,13 @@ function removeInternalAppFields(app) {
         health: app.health,
         location: app.location,
         accessRestriction: app.accessRestriction,
-        oauthProxy: app.oauthProxy,
         lastBackupId: app.lastBackupId,
         manifest: app.manifest,
         portBindings: app.portBindings,
         iconUrl: app.iconUrl,
-        fqdn: app.fqdn
+        fqdn: app.fqdn,
+        memoryLimit: app.memoryLimit,
+        altDomain: app.altDomain
     };
 }
 
@@ -64,19 +66,11 @@ function getApp(req, res, next) {
     });
 }
 
-function getAppBySubdomain(req, res, next) {
-    assert.strictEqual(typeof req.params.subdomain, 'string');
-
-    apps.getBySubdomain(req.params.subdomain, function (error, app) {
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such subdomain'));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(200, removeInternalAppFields(app)));
-    });
-}
-
 function getApps(req, res, next) {
-    apps.getAll(function (error, allApps) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    var func = req.user.admin ? apps.getAll : apps.getAllByUser.bind(null, req.user);
+    func(function (error, allApps) {
         if (error) return next(new HttpError(500, error));
 
         allApps = allApps.map(removeInternalAppFields);
@@ -95,80 +89,74 @@ function getAppIcon(req, res, next) {
     });
 }
 
-/*
- * Installs an app
- * @bodyparam {string} appStoreId The id of the app to be installed
- * @bodyparam {manifest} manifest The app manifest
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from environment variable name to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- * @bodyparam {icon} icon Base64 encoded image
- */
 function installApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId is required'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    // required
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (typeof data.oauthProxy !== 'boolean') return next(new HttpError(400, 'oauthProxy must be a boolean'));
+
+    // optional
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
 
-    // allow tests to provide an appId for testing
-    var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
+    if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
 
-    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j oauthproxy:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.oauthProxy, data.manifest);
+    // falsy value in altDomain unsets it
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.oauthProxy, data.icon || null, data.cert || null, data.key || null, function (error) {
+    debug('Installing app id:%s data:%j', data);
+
+    apps.install(data, auditSource(req), function (error, app) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202, { id: appId } ));
+        next(new HttpSuccess(202, app));
     });
 }
 
-/*
- * Configure an app
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from env to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- */
 function configureApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (typeof data.oauthProxy !== 'boolean') return next(new HttpError(400, 'oauthProxy must be a boolean'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if ('location' in data && typeof data.location !== 'string') return next(new HttpError(400, 'location must be string'));
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    if ('accessRestriction' in data && typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
 
-    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j oauthProxy:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.oauthProxy);
+    if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.oauthProxy, data.cert || null, data.key || null, function (error) {
+    debug('Configuring app id:%s data:%j', req.params.id, data);
+
+    apps.configure(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -183,14 +171,21 @@ function configureApp(req, res, next) {
 }
 
 function restoreApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
+    var data = req.body;
+
     debug('Restore app id:%s', req.params.id);
 
-    apps.restore(req.params.id, function (error) {
+    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
+    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+
+    apps.restore(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, { }));
@@ -212,16 +207,12 @@ function backupApp(req, res, next) {
     });
 }
 
-/*
- * Uninstalls an app
- * @bodyparam {string} id The id of the app to be uninstalled
- */
 function uninstallApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
     debug('Uninstalling app id:%s', req.params.id);
 
-    apps.uninstall(req.params.id, function (error) {
+    apps.uninstall(req.params.id, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error) return next(new HttpError(500, error));
 
@@ -263,15 +254,18 @@ function updateApp(req, res, next) {
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
     debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
 
-    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings, data.icon, function (error) {
+    apps.update(req.params.id, req.body, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -341,15 +335,37 @@ function getLogs(req, res, next) {
     });
 }
 
+function demuxStream(stream, stdin) {
+    var header = null;
+
+    stream.on('readable', function() {
+        header = header || stream.read(4);
+
+        while (header !== null) {
+            var length = header.readUInt32BE(0);
+            if (length === 0) {
+                header = null;
+                return stdin.end(); // EOF
+            }
+
+            var payload = stream.read(length);
+
+            if (payload === null) break;
+            stdin.write(payload);
+            header = stream.read(4);
+        }
+    });
+}
+
 function exec(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    debug('Execing into app id:%s', req.params.id);
+    debug('Execing into app id:%s and cmd:%s', req.params.id, req.query.cmd);
 
     var cmd = null;
     if (req.query.cmd) {
         cmd = safe.JSON.parse(req.query.cmd);
-        if (!util.isArray(cmd) && cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+        if (!util.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
     }
 
     var columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
@@ -370,15 +386,29 @@ function exec(req, res, next) {
         req.clearTimeout();
         res.sendUpgradeHandshake();
 
+        // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged.
         duplexStream.pipe(res.socket);
-        res.socket.pipe(duplexStream);
+
+        if (tty) {
+            res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit
+        } else {
+            demuxStream(res.socket, duplexStream);
+            res.socket.on('error', function () { duplexStream.end(); });
+            res.socket.on('end', function () { duplexStream.end(); });
+        }
     });
 }
 
 function listBackups(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.listBackups(req.params.id, function (error, result) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    apps.listBackups(page, perPage, req.params.id, function (error, result) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error) return next(new HttpError(500, error));
 

src/routes/backups.js

@@ -1,22 +1,30 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
     get: get,
-    create: create
+    create: create,
+    createDownloadUrl: createDownloadUrl
 };
 
-var backups = require('../backups.js'),
+var assert = require('assert'),
+    backups = require('../backups.js'),
     BackupsError = require('../backups.js').BackupsError,
-    cloudron = require('../cloudron.js'),
-    CloudronError = require('../cloudron.js').CloudronError,
-    debug = require('debug')('box:routes/backups'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function get(req, res, next) {
-    backups.getAllPaged(1, 5, function (error, result) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    backups.getPaged(page, perPage, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -27,10 +35,20 @@ function get(req, res, next) {
 function create(req, res, next) {
     // note that cloudron.backup only waits for backup initiation and not for backup to complete
     // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    backups.backup(auditSource(req), function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
     });
 }
+
+function createDownloadUrl(req, res, next) {
+    assert.strictEqual(typeof req.params.backupId, 'string');
+
+    backups.getRestoreUrl(req.params.backupId, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}

src/routes/clients.js

@@ -1,21 +1,19 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllByUserId: getAllByUserId,
+    getAll: getAll,
+    addClientToken: addClientToken,
     getClientTokens: getClientTokens,
-    delClientTokens: delClientTokens
+    delClientTokens: delClientTokens,
+    delToken: delToken
 };
 
 var assert = require('assert'),
-    clientdb = require('../clientdb.js'),
     clients = require('../clients.js'),
     ClientsError = clients.ClientsError,
-    DatabaseError = require('../databaseerror.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     validUrl = require('valid-url');
@@ -29,8 +27,8 @@ function add(req, res, next) {
     if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
     if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
 
-    clients.add(data.appId, clientdb.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, 'Invalid scope'));
+    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
+        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(201, result));
     });
@@ -40,7 +38,7 @@ function get(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
     clients.get(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'No such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, result));
     });
@@ -49,26 +47,49 @@ function get(req, res, next) {
 function del(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
-    clients.del(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+    clients.get(req.params.clientId, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(204, result));
+
+        // we do not allow to use the REST API to delete addon clients
+        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
+
+        clients.del(req.params.clientId, function (error, result) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+            if (error && error.reason === ClientsError.NOT_ALLOWED) return next(new HttpError(405, error.message));
+            if (error) return next(new HttpError(500, error));
+            next(new HttpSuccess(204, result));
+        });
     });
 }
 
-function getAllByUserId(req, res, next) {
-    clients.getAllWithDetailsByUserId(req.user.id, function (error, result) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return next(new HttpError(500, error));
+function getAll(req, res, next) {
+    clients.getAll(function (error, result) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { clients: result }));
     });
 }
 
+function addClientToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    var expiresAt = req.query.expiresAt ? parseInt(req.query.expiresAt, 10) : Date.now() + 24 * 60 * 60 * 1000; // default 1 day;
+    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
+
+    clients.addClientTokenByUserId(req.params.clientId, req.user.id, expiresAt, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+        next(new HttpSuccess(201, { token: result }));
+    });
+}
+
 function getClientTokens(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
     assert.strictEqual(typeof req.user, 'object');
 
     clients.getClientTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { tokens: result }));
     });
@@ -79,8 +100,22 @@ function delClientTokens(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
     clients.delClientTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(204));
     });
 }
+
+function delToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.params.tokenId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === ClientsError.INVALID_TOKEN) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/cloudron.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -10,20 +8,27 @@ exports = module.exports = {
     getProgress: getProgress,
     getConfig: getConfig,
     update: update,
-    migrate: migrate,
-    feedback: feedback
+    feedback: feedback,
+    checkForUpdates: checkForUpdates
 };
 
 var assert = require('assert'),
+    async = require('async'),
     cloudron = require('../cloudron.js'),
-    config = require('../config.js'),
-    progress = require('../progress.js'),
-    mailer = require('../mailer.js'),
     CloudronError = cloudron.CloudronError,
+    config = require('../config.js'),
     debug = require('debug')('box:routes/cloudron'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
-    superagent = require('superagent');
+    progress = require('../progress.js'),
+    mailer = require('../mailer.js'),
+    superagent = require('superagent'),
+    updateChecker = require('../updatechecker.js');
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
 
 /**
  * Creating an admin user and activate the cloudron.
@@ -51,11 +56,9 @@ function activate(req, res, next) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
     debug('activate: username:%s ip:%s', username, ip);
 
-    cloudron.activate(username, password, email, displayName, ip, function (error, info) {
+    cloudron.activate(username, password, email, displayName, ip, auditSource(req), function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
-        if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
-        if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
-        if (error && error.reason === CloudronError.BAD_EMAIL) return next(new HttpError(400, 'Bad email'));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         // only in caas case do we have to notify the api server about activation
@@ -120,7 +123,7 @@ function getConfig(req, res, next) {
 
 function update(req, res, next) {
     // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    cloudron.updateToLatest(auditSource(req), function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
@@ -129,24 +132,23 @@ function update(req, res, next) {
     });
 }
 
-function migrate(req, res, next) {
-    if (typeof req.body.size !== 'string') return next(new HttpError(400, 'size must be string'));
-    if (typeof req.body.region !== 'string') return next(new HttpError(400, 'region must be string'));
-
-    debug('Migration requested', req.body.size, req.body.region);
-
-    cloudron.migrate(req.body.size, req.body.region, function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(202, {}));
+function checkForUpdates(req, res, next) {
+    async.series([
+        updateChecker.checkAppUpdates,
+        updateChecker.checkBoxUpdates
+    ], function () {
+        next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
     });
 }
 
 function feedback(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK && req.body.type !== mailer.FEEDBACK_TYPE_TICKET && req.body.type !== mailer.FEEDBACK_TYPE_APP) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app"'));
+    if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK &&
+        req.body.type !== mailer.FEEDBACK_TYPE_TICKET &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_MISSING &&
+        req.body.type !== mailer.FEEDBACK_TYPE_UPGRADE_REQUEST &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback", "app_missing", "app_error" or "upgrade_request"'));
     if (typeof req.body.subject !== 'string' || !req.body.subject) return next(new HttpError(400, 'subject must be string'));
     if (typeof req.body.description !== 'string' || !req.body.description) return next(new HttpError(400, 'description must be string'));
 

src/routes/developer.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -15,8 +13,13 @@ var developer = require('../developer.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function enabled(req, res, next) {
-    developer.enabled(function (error, enabled) {
+    developer.isEnabled(function (error, enabled) {
         if (enabled) return next();
         next(new HttpError(412, 'Developer mode not enabled'));
     });
@@ -25,8 +28,9 @@ function enabled(req, res, next) {
 function setEnabled(req, res, next) {
     if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled must be boolean'));
 
-    developer.setEnabled(req.body.enabled, function (error) {
+    developer.setEnabled(req.body.enabled, auditSource(req), function (error) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, {}));
     });
 }
@@ -40,7 +44,7 @@ function login(req, res, next) {
         if (error) return next(new HttpError(500, error));
         if (!user) return next(new HttpError(401, 'Invalid credentials'));
 
-        developer.issueDeveloperToken(user, function (error, result) {
+        developer.issueDeveloperToken(user, auditSource(req), function (error, result) {
             if (error) return next(new HttpError(500, error));
 
             next(new HttpSuccess(200, { token: result.token, expiresAt: result.expiresAt }));
@@ -53,4 +57,4 @@ function apps(req, res, next) {
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { apps: result }));
     });
-}
+}

src/routes/eventlog.js

@@ -0,0 +1,26 @@
+'use strict';
+
+exports = module.exports = {
+    get: get
+};
+
+var eventlog = require('../eventlog.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
+
+function get(req, res, next) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    if (req.query.action && typeof req.query.action !== 'string') return next(new HttpError(400, 'action must be a string'));
+    if (req.query.search && typeof req.query.search !== 'string') return next(new HttpError(400, 'search must be a string'));
+
+    eventlog.getAllPaged(req.query.action || null, req.query.search || null, page, perPage, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { eventlogs: result }));
+    });
+}

src/routes/groups.js

@@ -0,0 +1,65 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    list: list,
+    create: create,
+    remove: remove
+};
+
+var assert = require('assert'),
+    groups = require('../groups.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    groups = require('../groups.js'),
+    GroupError = groups.GroupError;
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    groups.create(req.body.name, function (error, group) {
+        if (error && error.reason === GroupError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === GroupError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        var groupInfo = {
+            id: group.id,
+            name: group.name
+        };
+
+        next(new HttpSuccess(201, groupInfo));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.groupId, 'string');
+
+    groups.getWithMembers(req.params.groupId, function (error, result) {
+        if (error && error.reason === GroupError.NOT_FOUND) return next(new HttpError(404, 'No such group'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    groups.getAllWithMembers(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { groups: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.groupId, 'string');
+
+    groups.remove(req.params.groupId, function (error) {
+        if (error && error.reason === GroupError.NOT_FOUND) return next(new HttpError(404, 'Group not found'));
+        if (error && error.reason === GroupError.NOT_ALLOWED) return next(new HttpError(409, 'Group deletion not allowed'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/index.js

@@ -2,14 +2,17 @@
 
 exports = module.exports = {
     apps: require('./apps.js'),
+    backups: require('./backups.js'),
+    clients: require('./clients.js'),
     cloudron: require('./cloudron.js'),
     developer: require('./developer.js'),
+    eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
+    groups: require('./groups.js'),
+    mailboxes: require('./mailboxes.js'),
     oauth2: require('./oauth2.js'),
+    profile: require('./profile.js'),
+    sysadmin: require('./sysadmin.js'),
     settings: require('./settings.js'),
-    clients: require('./clients.js'),
-    backups: require('./backups.js'),
-    internal: require('./internal.js'),
     user: require('./user.js')
 };
-

src/routes/mailboxes.js

@@ -0,0 +1,91 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    remove: remove,
+    create: create,
+    setAliases: setAliases,
+    getAliases: getAliases
+};
+
+var assert = require('assert'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    mailboxes = require('../mailboxes.js'),
+    MailboxError = mailboxes.MailboxError,
+    util = require('util');
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    mailboxes.add(req.body.name, function (error, mailbox) {
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, mailbox));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.get(req.params.mailboxId, function (error, result) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { mailboxes: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.del(req.params.mailboxId, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'Mailbox not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    if (!util.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
+
+    for (var i = 0; i < req.body.aliases.length; i++) {
+        if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
+    }
+
+    mailboxes.setAliases(req.params.mailboxId, req.body.aliases, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'One or more alias already exist'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+function getAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.getAliases(req.params.mailboxId, function (error, aliases) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { aliases: aliases }));
+    });
+}

src/routes/oauth2.js

@@ -1,34 +1,37 @@
-/* jslint  node:true */
-
 'use strict';
 
-var assert = require('assert'),
+var appdb = require('../appdb'),
     apps = require('../apps'),
+    assert = require('assert'),
     authcodedb = require('../authcodedb'),
-    clientdb = require('../clientdb'),
+    clients = require('../clients'),
+    ClientsError = clients.ClientsError,
     config = require('../config.js'),
-    constants = require('../constants.js'),
     DatabaseError = require('../databaseerror'),
     debug = require('debug')('box:routes/oauth2'),
+    eventlog = require('../eventlog.js'),
+    hat = require('hat'),
     HttpError = require('connect-lastmile').HttpError,
     middleware = require('../middleware/index.js'),
     oauth2orize = require('oauth2orize'),
     passport = require('passport'),
     querystring = require('querystring'),
-    util = require('util'),
     session = require('connect-ensure-login'),
-    settings = require('../settings.js'),
     tokendb = require('../tokendb'),
-    appdb = require('../appdb'),
     url = require('url'),
     user = require('../user.js'),
     UserError = user.UserError,
-    hat = require('hat');
+    util = require('util'),
+    _ = require('underscore');
+
+function auditSource(req, appId) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { authType: 'oauth', ip: ip, appId: appId };
+}
 
 // create OAuth 2.0 server
 var gServer = oauth2orize.createServer();
 
-
 // Register serialialization and deserialization functions.
 //
 // The client id is stored in the session and can thus be retrieved for each
@@ -39,7 +42,7 @@ gServer.serializeClient(function (client, callback) {
 });
 
 gServer.deserializeClient(function (id, callback) {
-    clientdb.get(id, callback);
+    clients.get(id, callback);
 });
 
 
@@ -58,7 +61,7 @@ gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client,
     var code = hat(256);
     var expiresAt = Date.now() + 60 * 60000; // 1 hour
 
-    authcodedb.add(code, client.id, user.username, expiresAt, function (error) {
+    authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
         if (error) return callback(error);
 
         debug('grant code: new auth code for client %s code %s', client.id, code);
@@ -74,7 +77,7 @@ gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client,
     var token = tokendb.generateToken();
     var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_USER + user.id, client.id, expires, client.scope, function (error) {
+    tokendb.add(token, user.id, client.id, expires, client.scope, function (error) {
         if (error) return callback(error);
 
         debug('grant token: new access token for client %s token %s', client.id, token);
@@ -104,7 +107,7 @@ gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI,
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + authCode.userId, authCode.clientId, expires, client.scope, function (error) {
+            tokendb.add(token, authCode.userId, authCode.clientId, expires, client.scope, function (error) {
                 if (error) return callback(error);
 
                 debug('exchange: new access token for client %s token %s', client.id, token);
@@ -199,13 +202,13 @@ function loginForm(req, res) {
         });
     }
 
-    clientdb.get(u.query.client_id, function (error, result) {
+    clients.get(u.query.client_id, function (error, result) {
         if (error) return sendError(req, res, 'Unknown OAuth client');
 
         switch (result.type) {
-            case clientdb.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
+            case clients.TYPE_BUILT_IN: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
             default: break;
         }
 
@@ -268,19 +271,60 @@ function passwordSentSite(req, res) {
     renderTemplate(res, 'password_reset_sent', { adminOrigin: config.adminOrigin(), title: 'Cloudron Password Reset' });
 }
 
-// -> GET /api/v1/session/password/setup.html
-function passwordSetupSite(req, res, next) {
-    if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
+function renderAccountSetupSite(res, req, userObject, error) {
+    renderTemplate(res, 'account_setup', {
+        adminOrigin: config.adminOrigin(),
+        user: userObject,
+        error: error,
+        csrf: req.csrfToken(),
+        resetToken: req.query.reset_token || req.body.resetToken,
+        title: 'Cloudron Password Setup'
+    });
+}
 
-    user.getByResetToken(req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid reset_token'));
+// -> GET /api/v1/session/account/setup.html
+function accountSetupSite(req, res) {
+    if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');
 
-        renderTemplate(res, 'password_setup', {
-            adminOrigin: config.adminOrigin(),
-            user: user,
-            csrf: req.csrfToken(),
-            resetToken: req.query.reset_token,
-            title: 'Cloudron Password Setup'
+    user.getByResetToken(req.query.reset_token, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Reset Token');
+
+        renderAccountSetupSite(res, req, userObject, '');
+    });
+}
+
+// -> POST /api/v1/session/account/setup
+function accountSetup(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
+    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
+    if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
+
+    debug('acountSetup: with token %s.', req.body.resetToken);
+
+    user.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Reset Token');
+
+        var data = _.pick(req.body, 'username', 'displayName');
+        user.update(userObject.id, data, auditSource(req), function (error) {
+            if (error && error.reason === UserError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
+            if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+            if (error && error.reason === UserError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
+            if (error) return next(new HttpError(500, error));
+
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
+            // setPassword clears the resetToken
+            user.setPassword(userObject.id, req.body.password, function (error, result) {
+                if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+
+                if (error) return next(new HttpError(500, error));
+
+                res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
+            });
         });
     });
 }
@@ -314,9 +358,11 @@ function passwordReset(req, res, next) {
     user.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return next(new HttpError(401, 'Invalid resetToken'));
 
+        if (!userObject.username) return next(new HttpError(401, 'No username set'));
+
         // setPassword clears the resetToken
         user.setPassword(userObject.id, req.body.password, function (error, result) {
-            if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
+            if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(406, error.message));
             if (error) return next(new HttpError(500, error));
 
             res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -358,8 +404,8 @@ var authorization = [
     gServer.authorization({}, function (clientId, redirectURI, callback) {
         debug('authorization: client %s with callback to %s.', clientId, redirectURI);
 
-        clientdb.get(clientId, function (error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function (error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
             // ignore the origin passed into form the client, but use the one from the clientdb
@@ -373,16 +419,24 @@ var authorization = [
         // Handle our different types of oauth clients
         var type = req.oauth2.client.type;
 
-        if (type === clientdb.TYPE_ADMIN) return next();
-        if (type === clientdb.TYPE_EXTERNAL) return next();
-        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unkonwn OAuth client.');
+        if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
+            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id });
+            return next();
+        } else if (type === clients.TYPE_SIMPLE_AUTH) {
+            return sendError(req, res, 'Unknown OAuth client.');
+        }
 
         appdb.get(req.oauth2.client.appId, function (error, appObject) {
             if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
 
-            if (!apps.hasAccessTo(appObject, req.oauth2.user)) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
+            apps.hasAccessTo(appObject, req.oauth2.user, function (error, access) {
+                if (error) return sendError(req, res, 'Internal error');
+                if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
 
-            next();
+                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id });
+
+                next();
+            });
         });
     },
     gServer.decision({ loadTransaction: false })
@@ -402,6 +456,31 @@ var token = [
     gServer.errorHandler()
 ];
 
+// tests if all requestedScopes are attached to the request
+function validateRequestedScopes(req, requestedScopes) {
+    assert.strictEqual(typeof req, 'object');
+    assert(Array.isArray(requestedScopes));
+
+    if (!req.authInfo || !req.authInfo.scope) return new Error('No scope found');
+
+    var scopes = req.authInfo.scope.split(',');
+
+    // check for roles separately
+    if (requestedScopes.indexOf(clients.SCOPE_ROLE_SDK) !== -1 && scopes.indexOf(clients.SCOPE_ROLE_SDK) === -1) {
+        return new Error('Missing required scope role "' + clients.SCOPE_ROLE_SDK + '"');
+    }
+
+    if (scopes.indexOf('*') !== -1) return null;
+
+    for (var i = 0; i < requestedScopes.length; ++i) {
+        if (scopes.indexOf(requestedScopes[i]) === -1) {
+            debug('scope: missing scope "%s".', requestedScopes[i]);
+            return new Error('Missing required scope "' + requestedScopes[i] + '"');
+        }
+    }
+
+    return null;
+}
 
 //  The scope middleware provides an auth middleware for routes.
 //
@@ -421,17 +500,8 @@ function scope(requestedScope) {
     return [
         passport.authenticate(['bearer'], { session: false }),
         function (req, res, next) {
-            if (!req.authInfo || !req.authInfo.scope) return next(new HttpError(401, 'No scope found'));
-            if (req.authInfo.scope === '*') return next();
-
-            var scopes = req.authInfo.scope.split(',');
-
-            for (var i = 0; i < requestedScopes.length; ++i) {
-                if (scopes.indexOf(requestedScopes[i]) === -1) {
-                    debug('scope: missing scope "%s".', requestedScopes[i]);
-                    return next(new HttpError(401, 'Missing required scope "' + requestedScopes[i] + '"'));
-                }
-            }
+            var error = validateRequestedScopes(req, requestedScopes);
+            if (error) return next(new HttpError(401, error.message));
 
             next();
         }
@@ -457,10 +527,12 @@ exports = module.exports = {
     passwordResetRequest: passwordResetRequest,
     passwordSentSite: passwordSentSite,
     passwordResetSite: passwordResetSite,
-    passwordSetupSite: passwordSetupSite,
     passwordReset: passwordReset,
+    accountSetupSite: accountSetupSite,
+    accountSetup: accountSetup,
     authorization: authorization,
     token: token,
+    validateRequestedScopes: validateRequestedScopes,
     scope: scope,
     csrf: csrf
 };