add pagination

this signals that an update worked

CHANGES

@@ -444,6 +444,53 @@
 - Store the backup config for each backup. This will allow using multiple buckets/providers for backups simultaneously.
 - Fix SPF record check
 
+[0.10.4]
+- Fix restore for droplets in EU region
+
 [0.11.0]
 - Store backups in the same region as the Cloudron
+- Fix PCRE security issue (http://www.ubuntu.com/usn/usn-2943-1/)
+
+[0.11.1]
+- Improve the backup logic
+
+[0.11.2]
+- Allow users to choose a username on first sign up
+- Fix app graphs
+
+[0.12.0]
+- Fix upload of large backups
+- Postgres addon whitelists pg_trgm and hstore extensions
+- Suppress boring update emails from patch releases
+- Setup bounce alerts for emails
+- Query admin's name in activation wizard
+- Admin emails are now delivered as no-reply
+- Fix crash when user attempts to set a duplicate email
+- Improved mongodb crash recovery
+
+[0.12.1]
+- Fix crash when backing up apps
+
+[0.12.2]
+- Improved error handling for addons
+
+[0.12.3]
+- LDAP: Do not set sn attribute when user has no surname
+
+[0.12.4]
+- Install app only after platform is ready
+
+[0.12.5]
+- Get alerts for app task failures
+- Fix update issue when one or more apps are in failed state
+
+[0.12.6]
+- Allow setting an alternate external domain for apps
+
+[0.12.7]
+- Fix changing password
+
+[0.13.0]
+- Upgrade to ubuntu 16.04
+- Add event log
 

baseimage/digitalocean.sh

@@ -30,7 +30,7 @@ function create_droplet() {
     local box_name="$2"
 
     local image_region="sfo1"
-    local ubuntu_image_slug="ubuntu-15-10-x64"
+    local ubuntu_image_slug="ubuntu-16-04-x64"
     local box_size="512mb"
 
     local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"

baseimage/initializeBaseUbuntuImage.sh

@@ -17,7 +17,7 @@ function die {
     exit 1
 }
 
-[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
+[[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"
 
 if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
     source "${SOURCE_DIR}/INFRA_VERSION"
@@ -183,7 +183,7 @@ fi
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
-[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
+[[ "$(nginx -v 2>&1)" == *"nginx/1.10."* ]] || die "Expecting nginx version to be 1.10.x"
 
 echo "==== Install build-essential ===="
 apt-get -y install build-essential rcconf
@@ -191,11 +191,11 @@ apt-get -y install build-essential rcconf
 echo "==== Install mysql ===="
 debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
 debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-apt-get -y install mysql-server
-[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
+apt-get -y install mysql-server-5.7
+[[ "$(mysqld --version 2>&1)" == *"5.7."* ]] || die "Expecting mysql version to be 5.7.x"
 
-echo "==== Install pwgen ===="
-apt-get -y install pwgen
+echo "==== Install pwgen and swaks awscli ===="
+apt-get -y install pwgen swaks awscli
 
 echo "==== Install collectd ==="
 if ! apt-get install -y collectd collectd-utils; then

crashnotifier.js

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();

crashnotifierservice.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendFailureLogs(processName, { unit: processName });
+}
+
+main();

gulpfile.js

@@ -22,6 +22,7 @@ gulp.task('3rdparty', function () {
         'webadmin/src/3rdparty/**/*.otf',
         'webadmin/src/3rdparty/**/*.eot',
         'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.gif',
         'webadmin/src/3rdparty/**/*.ttf',
         'webadmin/src/3rdparty/**/*.woff',
         'webadmin/src/3rdparty/**/*.woff2'

installer/systemd/box-setup.sh

@@ -4,7 +4,6 @@ set -eu -o pipefail
 
 readonly USER_HOME="/home/yellowtent"
 readonly APPS_SWAP_FILE="/apps.swap"
-readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups
 readonly USER_DATA_FILE="/root/user_data.img"
 readonly USER_DATA_DIR="/home/yellowtent/data"
 
@@ -23,7 +22,6 @@ readonly swap_size="${physical_memory}" # if you change this, fix enoughResource
 readonly app_count=$((${physical_memory} / 200)) # estimated app count
 readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
 readonly disk_size=$((disk_size_gb * 1024))
-readonly backup_swap_size=1024
 readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
 readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
 
@@ -32,8 +30,7 @@ echo "Physical memory: ${physical_memory}"
 echo "Estimated app count: ${app_count}"
 echo "Disk size: ${disk_size}"
 
-# Allocate two sets of swap files - one for general app usage and another for backup
-# The backup swap is setup for swap on the fly by the backup scripts
+# Allocate swap for general app usage
 if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
     echo "Creating Apps swap file of size ${swap_size}M"
     fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
@@ -45,17 +42,8 @@ else
     echo "Apps Swap file already exists"
 fi
 
-if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then
-    echo "Creating Backup swap file of size ${backup_swap_size}M"
-    fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}"
-    chmod 600 "${BACKUP_SWAP_FILE}"
-    mkswap "${BACKUP_SWAP_FILE}"
-else
-    echo "Backups Swap file already exists"
-fi
-
 echo "Resizing data volume"
-home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
+home_data_size=$((disk_size - system_size - swap_size - ext4_reserved))
 echo "Resizing up btrfs user data to size ${home_data_size}M"
 umount "${USER_DATA_DIR}" || true
 # Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data

migrations/20160404052453-backups-drop-configJson.js

@@ -0,0 +1,17 @@
+var dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20160404191651-backups-rename-filename-to-id.js

@@ -0,0 +1,16 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE filename id VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups CHANGE id filename VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160405083451-users-drop-username-null.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL UNIQUE', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160419070047-apps-add-altDomain.js

@@ -0,0 +1,17 @@
+'use strict';
+
+var dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN altDomain VARCHAR(256)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN altDomain', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160430062446-eventlog-add-table.js

@@ -0,0 +1,24 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE eventlog(" +
+            "id VARCHAR(128) NOT NULL," +
+            "source JSON," +
+            "creationTime TIMESTAMP," +
+            "action VARCHAR(128) NOT NULL," +
+            "data JSON," +
+            "PRIMARY KEY (id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE eventlog', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -11,7 +11,7 @@
 
 CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(254) NOT NULL UNIQUE,
+    username VARCHAR(254) UNIQUE,
     email VARCHAR(254) NOT NULL UNIQUE,
     password VARCHAR(1024) NOT NULL,
     salt VARCHAR(512) NOT NULL,
@@ -65,6 +65,7 @@ CREATE TABLE IF NOT EXISTS apps(
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     memoryLimit BIGINT DEFAULT 0,
+    altDomain VARCHAR(256),
 
     lastBackupId VARCHAR(128),
     lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
@@ -99,12 +100,20 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     FOREIGN KEY(appId) REFERENCES apps(id));
 
 CREATE TABLE IF NOT EXISTS backups(
-    filename VARCHAR(128) NOT NULL, /* s3 url, currently this is also the id */
+    filename VARCHAR(128) NOT NULL,
     creationTime TIMESTAMP,
     version VARCHAR(128) NOT NULL, /* app version or box version */
     type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
     dependsOn VARCHAR(4096), /* comma separate list of objects this backup depends on */
     state VARCHAR(16) NOT NULL,
-    configJson TEXT, /* configuration - bucket, prefix, key, provider */
 
     PRIMARY KEY (filename));
+
+CREATE TABLE IF NOT EXISTS eventlog(
+    id VARCHAR(128) NOT NULL,
+    action VARCHAR(128) NOT NULL,
+    source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
+    data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP,
+
+    PRIMARY KEY (id));

package.json

@@ -14,10 +14,9 @@
   ],
   "dependencies": {
     "async": "^1.2.1",
-    "attempt": "^1.0.1",
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "bytes": "^2.1.0",
+    "bytes": "^2.3.0",
     "cloudron-manifestformat": "^2.3.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
@@ -28,17 +27,16 @@
     "csurf": "^1.6.6",
     "db-migrate": "^0.9.2",
     "debug": "^2.2.0",
-    "dockerode": "^2.2.2",
+    "dockerode": "^2.2.10",
     "ejs": "^2.2.4",
-    "ejs-cli": "^1.0.1",
+    "ejs-cli": "^1.2.0",
     "express": "^4.12.4",
     "express-session": "^1.11.3",
     "hat": "0.0.3",
     "json": "^9.0.3",
     "ldapjs": "^0.7.1",
-    "memorystream": "^0.3.0",
     "mime": "^1.3.4",
-    "morgan": "^1.6.0",
+    "morgan": "^1.7.0",
     "multiparty": "^4.1.2",
     "mysql": "^2.7.0",
     "native-dns": "^0.7.0",
@@ -60,15 +58,15 @@
     "semver": "^4.3.6",
     "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
-    "superagent": "^1.5.0",
+    "superagent": "^1.8.3",
     "supererror": "^0.7.1",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "tldjs": "^1.6.2",
     "underscore": "^1.7.0",
-    "ursa": "^0.9.1",
+    "ursa": "^0.9.3",
     "valid-url": "^1.0.9",
-    "validator": "^4.4.0",
-    "x509": "^0.2.2"
+    "validator": "^4.9.0",
+    "x509": "^0.2.4"
   },
   "devDependencies": {
     "apidoc": "*",

setup/INFRA_VERSION

@@ -3,16 +3,16 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=23
+INFRA_VERSION=27
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
 BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.10.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.8.0
+MYSQL_IMAGE=cloudron/mysql:0.11.0
+POSTGRESQL_IMAGE=cloudron/postgresql:0.9.0
+MONGODB_IMAGE=cloudron/mongodb:0.9.0
 REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.9.0
+MAIL_IMAGE=cloudron/mail:0.10.0
 GRAPHITE_IMAGE=cloudron/graphite:0.8.0
 
 MYSQL_REPO=cloudron/mysql

setup/argparser.sh

@@ -21,6 +21,7 @@ arg_backup_config=""
 arg_dns_config=""
 arg_update_config=""
 arg_provider=""
+arg_app_bundle=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -37,6 +38,9 @@ while true; do
         $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
 EOF
         # read possibly empty parameters here
+        arg_app_bundle=$(echo "$2" | $json appBundle)
+        [[ "${arg_app_bundle}" == "" ]] && arg_app_bundle="[]"
+
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
         arg_token=$(echo "$2" | $json token)

setup/container/mysql.cnf

@@ -4,4 +4,4 @@
 # http://bugs.mysql.com/bug.php?id=68514
 [mysqld]
 performance_schema=OFF
-max_connection=50
+max_connections=50

setup/container/sudoers

@@ -25,9 +25,6 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
 
-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
-
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 

setup/container/systemd/crashnotifier@.service

@@ -7,7 +7,7 @@ StopWhenUnneeded=false
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
-ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+ExecStart="/home/yellowtent/box/crashnotifierservice.js" %I
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 KillMode=process
 User=yellowtent

setup/start.sh

@@ -138,7 +138,7 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "admin@${arg_fqdn}",
+    "adminEmail": "\"Cloudron\" <no-reply@${arg_fqdn}>",
     "provider": "${arg_provider}",
     "database": {
         "hostname": "localhost",
@@ -146,7 +146,8 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "password": "${mysql_root_password}",
         "port": 3306,
         "name": "box"
-    }
+    },
+    "appBundle": ${arg_app_bundle}
 }
 CONF_END
 

setup/start/nginx/appconfig.ejs

@@ -18,9 +18,10 @@ server {
     # https://bettercrypto.org/static/applied-crypto-hardening.pdf
     # https://mozilla.github.io/server-side-tls/ssl-config-generator/
     # https://cipherli.st/
+    # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
     ssl_prefer_server_ciphers on;
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
     add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
 
     proxy_http_version 1.1;

setup/start/setup_infra.sh

@@ -20,6 +20,7 @@ fi
 
 echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
 
+# TODO: be nice and stop addons cleanly (example, shutdown commands)
 existing_containers=$(docker ps -qa)
 echo "Remove containers: ${existing_containers}"
 if [[ -n "${existing_containers}" ]]; then
@@ -42,11 +43,14 @@ if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | g
 fi
 
 # mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
+# MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
+# MAIL_DOMAIN is the domain for which this server is relaying mails
 mail_container_id=$(docker run --restart=always -d --name="mail" \
     -m 75m \
     --memory-swap 150m \
     -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
+    -e "MAIL_SERVER_NAME=${arg_fqdn}" \
+    -e "MAIL_DOMAIN=${arg_fqdn}" \
     -v "${DATA_DIR}/box/mail:/app/data" \
     --read-only -v /tmp -v /run \
     "${MAIL_IMAGE}")

src/addons.js

@@ -19,22 +19,20 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    child_process = require('child_process'),
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:addons'),
-    docker = require('./docker.js').connection,
+    docker = require('./docker.js'),
+    dockerConnection = docker.connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
-    MemoryStream = require('memorystream'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
-    spawn = child_process.spawn,
     util = require('util'),
     uuid = require('node-uuid');
 
@@ -420,31 +418,14 @@ function setupMySql(app, options, callback) {
 
     debugApp(app, 'Setting up mysql');
 
-    var container = docker.getContainer('mysql');
     var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mysql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mysql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mysql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mysql', env, callback);
     });
 }
 
@@ -453,24 +434,14 @@ function teardownMySql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('mysql');
     var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];
 
     debugApp(app, 'Tearing down mysql');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mysql', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mysql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mysql', callback);
     });
 }
 
@@ -482,15 +453,9 @@ function backupMySql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mysql', cmd, { stdout: output }, callback);
 }
 
 function restoreMySql(app, options, callback) {
@@ -504,17 +469,8 @@ function restoreMySql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMySql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ];
+        docker.execContainer('mysql', cmd, { stdin: input }, callback);
     });
 }
 
@@ -525,31 +481,14 @@ function setupPostgreSql(app, options, callback) {
 
     debugApp(app, 'Setting up postgresql');
 
-    var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting postgresql addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'postgresql', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting postgresql addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'postgresql', env, callback);
     });
 }
 
@@ -558,24 +497,14 @@ function teardownPostgreSql(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
 
     debugApp(app, 'Tearing down postgresql');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('postgresql', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'postgresql', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'postgresql', callback);
     });
 }
 
@@ -587,19 +516,13 @@ function backupPostgreSql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/postgresql/service.sh', 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('postgresql', cmd, { stdout: output }, callback);
 }
 
 function restorePostgreSql(app, options, callback) {
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    callback = once(callback);
 
     setupPostgreSql(app, options, function (error) {
         if (error) return callback(error);
@@ -609,17 +532,9 @@ function restorePostgreSql(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
-        });
+        var cmd = [ '/addons/postgresql/service.sh', 'restore', app.id ];
 
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        docker.execContainer('postgresql', cmd, { stdin: input }, callback);
     });
 }
 
@@ -630,31 +545,14 @@ function setupMongoDb(app, options, callback) {
 
     debugApp(app, 'Setting up mongodb');
 
-    var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { bufferStdout: true }, function (error, stdout) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var stdout = new MemoryStream();
-            var stderr = new MemoryStream();
-
-            execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
-
-            var chunks = [ ];
-            stdout.on('data', function (chunk) { chunks.push(chunk); });
-
-            stream.on('error', callback);
-            stream.on('end', function () {
-                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debugApp(app, 'Setting mongodb addon config to %j', env);
-                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
-            });
-        });
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting mongodb addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'mongodb', env, callback);
     });
 }
 
@@ -663,24 +561,14 @@ function teardownMongoDb(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
 
     debugApp(app, 'Tearing down mongodb');
 
-    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+    docker.execContainer('mongodb', cmd, { }, function (error) {
         if (error) return callback(error);
 
-        execContainer.start(function (error, stream) {
-            if (error) return callback(error);
-
-            var data = '';
-            stream.on('error', callback);
-            stream.on('data', function (d) { data += d.toString('utf8'); });
-            stream.on('end', function () {
-                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
-            });
-        });
+        appdb.unsetAddonConfig(app.id, 'mongodb', callback);
     });
 }
 
@@ -692,15 +580,9 @@ function backupMongoDb(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
-        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
-    });
+    var cmd = [ '/addons/mongodb/service.sh', 'backup', app.id ];
 
-    cp.stdout.pipe(output);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('mongodb', cmd, { stdout: output }, callback);
 }
 
 function restoreMongoDb(app, options, callback) {
@@ -714,26 +596,16 @@ function restoreMongoDb(app, options, callback) {
         var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
         input.on('error', callback);
 
-        // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
-        cp.on('error', callback);
-        cp.on('exit', function (code, signal) {
-            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
-            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
-        });
-
-        cp.stdout.pipe(process.stdout);
-        cp.stderr.pipe(process.stderr);
-        input.pipe(cp.stdin).on('error', callback);
+        var cmd = [ '/addons/mongodb/service.sh', 'restore', app.id ];
+        docker.execContainer('mongodb', cmd, { stdin: input }, callback);
     });
 }
 
-
 function forwardRedisPort(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    docker.getContainer('redis-' + appId).inspect(function (error, data) {
+    dockerConnection.getContainer('redis-' + appId).inspect(function (error, data) {
         if (error) return callback(new Error('Unable to inspect container:' + error));
 
         var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
@@ -813,9 +685,9 @@ function setupRedis(app, options, callback) {
         'REDIS_PORT=6379'
     ];
 
-    var redisContainer = docker.getContainer(createOptions.name);
+    var redisContainer = dockerConnection.getContainer(createOptions.name);
     stopAndRemoveRedis(redisContainer, function () {
-        docker.createContainer(createOptions, function (error) {
+        dockerConnection.createContainer(createOptions, function (error) {
             if (error && error.statusCode !== 409) return callback(error); // if not already created
 
             redisContainer.start(function (error) {
@@ -836,7 +708,7 @@ function teardownRedis(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-   var container = docker.getContainer('redis-' + app.id);
+   var container = dockerConnection.getContainer('redis-' + app.id);
 
    var removeOptions = {
        force: true, // kill container if it's running
@@ -859,15 +731,7 @@ function teardownRedis(app, options, callback) {
 function backupRedis(app, options, callback) {
     debugApp(app, 'Backing up redis');
 
-    callback = once(callback); // ChildProcess exit may or may not be called after error
+    var cmd = [ '/addons/redis/service.sh', 'backup' ]; // the redis dir is volume mounted
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
-    cp.on('error', callback);
-    cp.on('exit', function (code, signal) {
-        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
-        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
-    });
-
-    cp.stdout.pipe(process.stdout);
-    cp.stderr.pipe(process.stderr);
+    docker.execContainer('redis-' + app.id, cmd, { }, callback);
 }

src/appdb.js

@@ -59,7 +59,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -177,7 +177,7 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, callback) {
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, altDomain, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
@@ -186,6 +186,7 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof accessRestriction, 'object');
     assert.strictEqual(typeof memoryLimit, 'number');
+    assert(altDomain === null || typeof altDomain === 'string');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
@@ -195,8 +196,8 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit, altDomain ]
     });
 
     Object.keys(portBindings).forEach(function (env) {

src/apphealthmonitor.js

@@ -162,7 +162,7 @@ function processDockerEvents() {
                 debug('OOM Context: %s', context);
 
                 // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+                if (error || app.appStoreId !== '') mailer.unexpectedExit(program, context); // app can be null if it's an addon crash
             });
         });
 

src/apps.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -18,12 +16,10 @@ exports = module.exports = {
     uninstall: uninstall,
 
     restore: restore,
-    restoreApp: restoreApp,
 
     update: update,
 
     backup: backup,
-    backupApp: backupApp,
     listBackups: listBackups,
 
     getLogs: getLogs,
@@ -35,8 +31,6 @@ exports = module.exports = {
 
     checkManifestConstraints: checkManifestConstraints,
 
-    setRestorePoint: setRestorePoint,
-
     autoupdateApps: autoupdateApps,
 
     // exported for testing
@@ -50,13 +44,13 @@ var addons = require('./addons.js'),
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
     certificates = require('./certificates.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apps'),
     docker = require('./docker.js'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     groups = require('./groups.js'),
     manifestFormat = require('cloudron-manifestformat'),
@@ -64,7 +58,6 @@ var addons = require('./addons.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
-    shell = require('./shell.js'),
     spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
@@ -72,26 +65,6 @@ var addons = require('./addons.js'),
     util = require('util'),
     validator = require('validator');
 
-var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
-    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
-
-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 // http://dustinsenos.com/articles/customErrorsInNode
 // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
 function AppsError(reason, errorOrMessage) {
@@ -160,7 +133,7 @@ function validatePortBindings(portBindings, tcpPorts) {
         2004, /* graphite (lo) */
         2020, /* install server */
         config.get('port'), /* app server (lo) */
-        config.get('internalPort'), /* internal app server (lo) */
+        config.get('sysadminPort'), /* sysadmin app server (lo) */
         config.get('ldapPort'), /* ldap server (lo) */
         config.get('oauthProxyPort'), /* oauth proxy server (lo) */
         config.get('simpleAuthPort'), /* simple auth server (lo) */
@@ -289,7 +262,7 @@ function get(appId, callback) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
         app.iconUrl = getIconUrlSync(app);
-        app.fqdn = config.appFqdn(app.location);
+        app.fqdn = app.altDomain || config.appFqdn(app.location);
 
         callback(null, app);
     });
@@ -304,7 +277,7 @@ function getBySubdomain(subdomain, callback) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
         app.iconUrl = getIconUrlSync(app);
-        app.fqdn = config.appFqdn(app.location);
+        app.fqdn = app.altDomain || config.appFqdn(app.location);
 
         callback(null, app);
     });
@@ -322,7 +295,7 @@ function getByIpAddress(ip, callback) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             app.iconUrl = getIconUrlSync(app);
-            app.fqdn = config.appFqdn(app.location);
+            app.fqdn = app.altDomain || config.appFqdn(app.location);
 
             callback(null, app);
         });
@@ -337,7 +310,7 @@ function getAll(callback) {
 
         apps.forEach(function (app) {
             app.iconUrl = getIconUrlSync(app);
-            app.fqdn = config.appFqdn(app.location);
+            app.fqdn = app.altDomain || config.appFqdn(app.location);
         });
 
         callback(null, apps);
@@ -381,7 +354,7 @@ function purchase(appStoreId, callback) {
     });
 }
 
-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, callback) {
+function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, altDomain, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
@@ -392,6 +365,8 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     assert(cert === null || typeof cert === 'string');
     assert(key === null || typeof key === 'string');
     assert.strictEqual(typeof memoryLimit, 'number');
+    assert(altDomain === null || typeof altDomain === 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var error = manifestFormat.parse(manifest);
@@ -415,6 +390,8 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     // memoryLimit might come in as 0 if not specified
     memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
 
+    if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+
     // singleUser mode requires accessRestriction to contain exactly one user
     if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
     if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
@@ -435,7 +412,7 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     purchase(appStoreId, function (error) {
         if (error) return callback(error);
 
-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, function (error) {
+        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, altDomain, function (error) {
             if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -447,12 +424,14 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
 
             taskmanager.restartAppTask(appId);
 
+            eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+
             callback(null);
         });
     });
 }
 
-function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, callback) {
+function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, altDomain, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
@@ -460,6 +439,8 @@ function configure(appId, location, portBindings, accessRestriction, cert, key,
     assert(cert === null || typeof cert === 'string');
     assert(key === null || typeof key === 'string');
     assert.strictEqual(typeof memoryLimit, 'number');
+    assert(altDomain === null || typeof altDomain === 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var error = validateHostname(location, config.fqdn());
@@ -471,6 +452,8 @@ function configure(appId, location, portBindings, accessRestriction, cert, key,
     error = certificates.validateCertificate(cert, key, config.appFqdn(location));
     if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
 
+    if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -495,12 +478,14 @@ function configure(appId, location, portBindings, accessRestriction, cert, key,
             accessRestriction: accessRestriction,
             portBindings: portBindings,
             memoryLimit: memoryLimit,
+            altDomain: altDomain,
 
             oldConfig: {
                 location: app.location,
                 accessRestriction: app.accessRestriction,
                 portBindings: app.portBindings,
-                memoryLimit: app.memoryLimit
+                memoryLimit: app.memoryLimit,
+                altDomain: altDomain
             }
         };
 
@@ -513,17 +498,20 @@ function configure(appId, location, portBindings, accessRestriction, cert, key,
 
             taskmanager.restartAppTask(appId);
 
+            eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId: appId });
+
             callback(null);
         });
     });
 }
 
-function update(appId, force, manifest, portBindings, icon, callback) {
+function update(appId, force, manifest, portBindings, icon, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof force, 'boolean');
     assert(manifest && typeof manifest === 'object');
-    assert(!portBindings || typeof portBindings === 'object');
+    assert(typeof portBindings === 'object'); // can be null
     assert(!icon || typeof icon === 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Will update app with id:%s', appId);
@@ -572,7 +560,8 @@ function update(appId, force, manifest, portBindings, icon, callback) {
                 manifest: app.manifest,
                 portBindings: app.portBindings,
                 accessRestriction: app.accessRestriction,
-                memoryLimit: app.memoryLimit
+                memoryLimit: app.memoryLimit,
+                altDomain: app.altDomain
             }
         };
 
@@ -583,6 +572,8 @@ function update(appId, force, manifest, portBindings, icon, callback) {
 
             taskmanager.restartAppTask(appId);
 
+            eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest });
+
             callback(null);
         });
     });
@@ -633,8 +624,9 @@ function getLogs(appId, lines, follow, callback) {
     });
 }
 
-function restore(appId, callback) {
+function restore(appId, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Will restore app with id:%s', appId);
@@ -665,7 +657,8 @@ function restore(appId, callback) {
                     accessRestriction: app.accessRestriction,
                     portBindings: app.portBindings,
                     memoryLimit: app.memoryLimit,
-                    manifest: app.manifest
+                    manifest: app.manifest,
+                    altDomain: app.altDomain
                 }
             };
         }
@@ -676,13 +669,16 @@ function restore(appId, callback) {
 
             taskmanager.restartAppTask(appId);
 
+            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { appId: appId });
+
             callback(null);
         });
     });
 }
 
-function uninstall(appId, callback) {
+function uninstall(appId, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Will uninstall app with id:%s', appId);
@@ -692,6 +688,8 @@ function uninstall(appId, callback) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
+            eventlog.add(eventlog.ACTION_APP_UNINSTALL, auditSource, { appId: appId });
+
             taskmanager.startAppTask(appId, callback);
         });
     });
@@ -789,20 +787,6 @@ function exec(appId, options, callback) {
     });
 }
 
-function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof lastBackupId, 'string');
-    assert.strictEqual(typeof lastBackupConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        return callback(null);
-    });
-}
-
 function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
     assert.strictEqual(typeof updateInfo, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -847,97 +831,6 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     }, callback);
 }
 
-function canBackupApp(app) {
-    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
-    // state not good for consistent backup (i.e addons may not have been setup completely)
-    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
-            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
-            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
-            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
-}
-
-// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
-// s3 does not allow changing creation time, so copying the last backup is easy way out for now
-function reuseOldBackup(app, callback) {
-    assert.strictEqual(typeof app.lastBackupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    backups.copyLastBackup(app, function (error, newBackupId) {
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
-
-        callback(null, newBackupId);
-    });
-}
-
-function createNewBackup(app, addonsToBackup, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    backups.getAppBackupUrl(app, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        debugApp(app, 'backupApp: backup url:%s backup config url:%s', result.url, result.configUrl);
-
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.configUrl, result.backupKey, result.sessionToken ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            callback(null, result.id);
-        });
-    });
-}
-
-function backupApp(app, addonsToBackup, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var appConfig = null, backupFunction;
-
-    if (!canBackupApp(app)) {
-        if (!app.lastBackupId) {
-            debugApp(app, 'backupApp: cannot backup app');
-            return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy and never backed up previously'));
-        }
-
-        appConfig = app.lastBackupConfig;
-        backupFunction = reuseOldBackup.bind(null, app);
-    } else {
-        appConfig = {
-            manifest: app.manifest,
-            location: app.location,
-            portBindings: app.portBindings,
-            accessRestriction: app.accessRestriction,
-            memoryLimit: app.memoryLimit
-        };
-        backupFunction = createNewBackup.bind(null, app, addonsToBackup);
-
-        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
-            return callback(safe.error);
-        }
-    }
-
-    backupFunction(function (error, backupId) {
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        debugApp(app, 'backupApp: successful id:%s', backupId);
-
-        setRestorePoint(app.id, backupId, appConfig, function (error) {
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            return callback(null, backupId);
-        });
-    });
-}
-
 function backup(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -957,26 +850,6 @@ function backup(appId, callback) {
     });
 }
 
-function restoreApp(app, addonsToRestore, backupId, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToRestore, 'object');
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-    assert(app.lastBackupId);
-
-    backups.getRestoreUrl(backupId, function (error, result) {
-        if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
-
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            addons.restoreAddons(app, addonsToRestore, callback);
-        });
-    });
-}
 
 function listBackups(page, perPage, appId, callback) {
     assert(typeof page === 'number' && page > 0);

src/apptask.js

@@ -1,7 +1,5 @@
 #!/usr/bin/env node
 
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -19,7 +17,8 @@ exports = module.exports = {
     _verifyManifest: verifyManifest,
     _registerSubdomain: registerSubdomain,
     _unregisterSubdomain: unregisterSubdomain,
-    _waitForDnsPropagation: waitForDnsPropagation
+    _waitForDnsPropagation: waitForDnsPropagation,
+    _waitForAltDomainDnsPropagation: waitForAltDomainDnsPropagation
 };
 
 require('supererror')({ splatchError: true });
@@ -35,6 +34,7 @@ var addons = require('./addons.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
+    backups = require('./backups.js'),
     certificates = require('./certificates.js'),
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
@@ -58,6 +58,7 @@ var addons = require('./addons.js'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
     uuid = require('node-uuid'),
+    waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
 var COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
@@ -100,9 +101,7 @@ function configureNginx(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var vhost = config.appFqdn(app.location);
-
-    certificates.ensureCertificate(vhost, function (error, certFilePath, keyFilePath) {
+    certificates.ensureCertificate(app, function (error, certFilePath, keyFilePath) {
         if (error) return callback(error);
 
         nginx.configureApp(app, certFilePath, keyFilePath, callback);
@@ -231,17 +230,19 @@ function downloadIcon(app, callback) {
 
     var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
 
-    superagent
-        .get(iconUrl)
-        .buffer(true)
-        .end(function (error, res) {
-            if (error && !error.response) return callback(new Error('Network error downloading icon:' + error.message));
-            if (res.statusCode !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        superagent
+            .get(iconUrl)
+            .buffer(true)
+            .end(function (error, res) {
+                if (error && !error.response) return retryCallback(new Error('Network error downloading icon:' + error.message));
+                if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli
 
-            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return retryCallback(new Error('Error saving icon:' + safe.error.message));
 
-            callback(null);
-    });
+                retryCallback(null);
+        });
+    }, callback);
 }
 
 function registerSubdomain(app, callback) {
@@ -318,20 +319,23 @@ function waitForDnsPropagation(app, callback) {
         return callback(null);
     }
 
-    function retry(error) {
-        debugApp(app, 'waitForDnsPropagation: ', error);
-        setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
-    }
+    async.retry({ interval: 5000, times: 120 }, function checkStatus(retryCallback) {
+        subdomains.status(app.dnsRecordId, function (error, result) {
+            if (error) return retryCallback(new Error('Failed to get dns record status : ' + error.message));
 
-    subdomains.status(app.dnsRecordId, function (error, result) {
-        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
+            if (result !== 'done') return retryCallback(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
+            retryCallback(null, result);
+        });
+    }, callback);
+}
 
-        callback(null);
-    });
+function waitForAltDomainDnsPropagation(app, callback) {
+    if (!app.altDomain) return callback(null);
+
+    waitForDns(app.altDomain, config.appFqdn(app.location), 'CNAME', callback); // waits forever
 }
 
 // updates the app object and the database
@@ -410,9 +414,12 @@ function install(app, callback) {
 
         runApp.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
+
         updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
         configureNginx.bind(null, app),
 
@@ -436,7 +443,7 @@ function backup(app, callback) {
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        apps.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest.addons),
 
         // done!
         function (callback) {
@@ -501,7 +508,7 @@ function restore(app, callback) {
         createVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons, backupId),
+        backups.restoreApp.bind(null, app, app.manifest.addons, backupId),
 
         updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
         createContainer.bind(null, app),
@@ -511,9 +518,12 @@ function restore(app, callback) {
 
         runApp.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
         configureNginx.bind(null, app),
 
@@ -573,6 +583,9 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for Alt Domain DNS propagation' }),
+        exports._waitForAltDomainDnsPropagation.bind(null, app),
+
         updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
         configureNginx.bind(null, app),
 
@@ -629,7 +642,7 @@ function update(app, callback) {
 
             async.series([
                 updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
-                apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                backups.backupApp.bind(null, app, app.oldConfig.manifest.addons)
             ], next);
         },
 

src/auth.js

@@ -28,11 +28,11 @@ function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     passport.serializeUser(function (user, callback) {
-        callback(null, user.username);
+        callback(null, user.id);
     });
 
-    passport.deserializeUser(function(username, callback) {
-        userdb.get(username, function (error, result) {
+    passport.deserializeUser(function(userId, callback) {
+        userdb.get(userId, function (error, result) {
             if (error) return callback(error);
 
             var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -44,7 +44,7 @@ function initialize(callback) {
 
     passport.use(new LocalStrategy(function (username, password, callback) {
         if (username.indexOf('@') === -1) {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                 if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                 if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                 if (error) return callback(error);
@@ -74,7 +74,7 @@ function initialize(callback) {
                 return callback(null, client);
             });
         } else {
-            user.verify(username, password, function (error, result) {
+            user.verifyWithUsername(username, password, function (error, result) {
                 if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
                 if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
                 if (error) return callback(error);

src/backupdb.js

@@ -3,10 +3,9 @@
 var assert = require('assert'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror.js'),
-    safe = require('safetydance'),
     util = require('util');
 
-var BACKUPS_FIELDS = [ 'filename', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'configJson' ];
+var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', ];
 
 exports = module.exports = {
     add: add,
@@ -27,8 +26,6 @@ function postProcess(result) {
     assert.strictEqual(typeof result, 'object');
 
     result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
-    result.config = safe.JSON.parse(result.configJson);
-    delete result.configJson;
 }
 
 function getPaged(page, perPage, callback) {
@@ -52,7 +49,7 @@ function getByAppIdPaged(page, perPage, appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND filename LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
         [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, 'appbackup\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
@@ -62,12 +59,12 @@ function getByAppIdPaged(page, perPage, appId, callback) {
     });
 }
 
-function get(filename, callback) {
-    assert.strictEqual(typeof filename, 'string');
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE filename = ? ORDER BY creationTime DESC',
-        [ filename ], function (error, result) {
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE id = ? ORDER BY creationTime DESC',
+        [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
@@ -79,17 +76,16 @@ function get(filename, callback) {
 
 function add(backup, callback) {
     assert(backup && typeof backup === 'object');
-    assert.strictEqual(typeof backup.filename, 'string');
+    assert.strictEqual(typeof backup.id, 'string');
     assert.strictEqual(typeof backup.version, 'string');
     assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
     assert(util.isArray(backup.dependsOn));
-    assert(backup.config && typeof backup.config === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var creationTime = backup.creationTime || new Date(); // allow tests to set the time
 
-    database.query('INSERT INTO backups (filename, version, type, creationTime, state, dependsOn, configJson) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        [ backup.filename, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(','), JSON.stringify(backup.config) ],
+    database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn) VALUES (?, ?, ?, ?, ?, ?)',
+        [ backup.id, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(',') ],
         function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
@@ -107,11 +103,11 @@ function clear(callback) {
     });
 }
 
-function del(filename, callback) {
-    assert.strictEqual(typeof filename, 'string');
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM backups WHERE filename=?', [ filename ], function (error) {
+    database.query('DELETE FROM backups WHERE id=?', [ id ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         callback(null);
     });

src/backups.js

@@ -6,22 +6,51 @@ exports = module.exports = {
     getPaged: getPaged,
     getByAppIdPaged: getByAppIdPaged,
 
-    getBackupUrl: getBackupUrl,
-    getAppBackupUrl: getAppBackupUrl,
     getRestoreUrl: getRestoreUrl,
 
-    copyLastBackup: copyLastBackup
+    ensureBackup: ensureBackup,
+
+    backup: backup,
+    backupApp: backupApp,
+    restoreApp: restoreApp,
+
+    backupBoxAndApps: backupBoxAndApps
 };
 
-var assert = require('assert'),
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    async = require('async'),
+    assert = require('assert'),
     backupdb = require('./backupdb.js'),
     caas = require('./storage/caas.js'),
     config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:backups'),
+    eventlog = require('./eventlog.js'),
+    locker = require('./locker.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    progress = require('./progress.js'),
     s3 = require('./storage/s3.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
     settings = require('./settings.js'),
     util = require('util'),
-    _ = require('underscore');
+    webhooks = require('./webhooks.js');
+
+var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
 
 function BackupsError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
@@ -44,6 +73,7 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.BAD_STATE = 'bad state';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';
 
 // choose which storage backend we use for test purpose we use s3
@@ -67,11 +97,6 @@ function getPaged(page, perPage, callback) {
     });
 }
 
-// this should probably be provider specific
-function cleanBackupConfig(backupConfig) {
-    return _.pick(backupConfig, 'provider', 'key', 'bucket', 'prefix', 'region');
-}
-
 function getByAppIdPaged(page, perPage, appId, callback) {
     assert(typeof page === 'number' && page > 0);
     assert(typeof perPage === 'number' && perPage > 0);
@@ -85,7 +110,7 @@ function getByAppIdPaged(page, perPage, appId, callback) {
     });
 }
 
-function getBackupUrl(appBackupIds, callback) {
+function getBoxBackupCredentials(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
@@ -96,62 +121,42 @@ function getBackupUrl(appBackupIds, callback) {
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
             if (error) return callback(error);
 
-            var obj = {
-                id: filename,
-                url: result.url,
-                sessionToken: result.sessionToken,
-                backupKey: backupConfig.key
-            };
+            result.id = filename;
+            result.s3Url = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + filename;
+            result.backupKey = backupConfig.key;
 
-            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            debug('getBoxBackupCredentials: %j', result);
 
-            backupdb.add({ filename: filename, creationTime: now, version: config.version(), type: backupdb.BACKUP_TYPE_BOX,
-                           dependsOn: appBackupIds, config: cleanBackupConfig(backupConfig) }, function (error) {
-                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-                callback(null, obj);
-            });
+            callback(null, result);
         });
     });
 }
 
-function getAppBackupUrl(app, callback) {
+function getAppBackupCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var now = new Date();
     var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
     var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
- 
+
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, configFilename, function (error, configResult) {
+        api(backupConfig.provider).getBackupCredentials(backupConfig, function (error, result) {
             if (error) return callback(error);
 
-            api(backupConfig.provider).getSignedUploadUrl(backupConfig, dataFilename, function (error, dataResult) {
-                if (error) return callback(error);
+            result.id = dataFilename;
+            result.s3ConfigUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + configFilename;
+            result.s3DataUrl = 's3://' + backupConfig.bucket + '/' + backupConfig.prefix + '/' + dataFilename;
+            result.backupKey = backupConfig.key;
 
-                var obj = {
-                    id: dataFilename,
-                    url: dataResult.url,
-                    configUrl: configResult.url,
-                    sessionToken: dataResult.sessionToken, // this token can be used for both config and data upload
-                    backupKey: backupConfig.key // only data is encrypted
-                };
+            debug('getAppBackupCredentials: %j', result);
 
-                debug('getAppBackupUrl: %j', obj);
-
-                backupdb.add({ filename: dataFilename, creationTime: now, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP,
-                               dependsOn: [ ], config: cleanBackupConfig(backupConfig) }, function (error) {
-                    if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-                    callback(null, obj);
-                });
-            });
+            callback(null, result);
         });
     });
 }
@@ -161,27 +166,21 @@ function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    settings.getBackupConfig(function (error, apiConfig) {
+    settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        backupdb.get(backupId, function (error, result) {
-            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+        api(backupConfig.provider).getRestoreUrl(backupConfig, backupId, function (error, result) {
+            if (error) return callback(error);
 
-            var backupConfig = result.config;
-            api(backupConfig.provider).getSignedDownloadUrl(apiConfig, backupConfig, backupId, function (error, result) {
-                if (error) return callback(error);
+            var obj = {
+                id: backupId,
+                url: result.url,
+                backupKey: backupConfig.key
+            };
 
-                var obj = {
-                    id: backupId,
-                    url: result.url,
-                    sessionToken: result.sessionToken,
-                    backupKey: backupConfig.key
-                };
+            debug('getRestoreUrl: id:%s url:%s backupKey:%s', obj.id, obj.url, obj.backupKey);
 
-                debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
-
-                callback(null, obj);
-            });
+            callback(null, obj);
         });
     });
 }
@@ -191,18 +190,23 @@ function copyLastBackup(app, callback) {
     assert.strictEqual(typeof app.lastBackupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, (new Date()).toISOString(), app.manifest.version);
+    var now = new Date();
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), app.manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), app.manifest.version);
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
+        debug('copyLastBackup: copying archive %s to %s', app.lastBackupId, toFilenameArchive);
+
         api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
             if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
             // TODO change that logic by adjusting app.lastBackupId to not contain the file type
             var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';
 
+            debug('copyLastBackup: copying config %s to %s', configFileId, toFilenameConfig);
+
             api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
                 if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
@@ -211,3 +215,257 @@ function copyLastBackup(app, callback) {
         });
     });
 }
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    getBoxBackupCredentials(appBackupIds, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        debug('backupBoxWithAppBackupIds:  %j', result);
+
+        var args = [ result.s3Url, result.accessKeyId, result.secretAccessKey, result.sessionToken, result.region, result.backupKey ];
+
+        shell.sudo('backupBox', [ BACKUP_BOX_CMD ].concat(args), function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debug('backupBoxWithAppBackupIds: success');
+
+            backupdb.add({ id: result.id, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                    if (error) return callback(error);
+                    callback(null, result.id);
+                });
+            });
+        });
+    });
+}
+
+// this function expects you to have a lock
+// function backupBox(callback) {
+//    apps.getAll(function (error, allApps) {
+//         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+//
+//         var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+//         appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+//
+//         backupBoxWithAppBackupIds(appBackupIds, callback);
+//     });
+// }
+
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldAppBackup(app, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    copyLastBackup(app, function (error, newBackupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewAppBackup(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAppBackupCredentials(app, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
+
+        var args = [ app.id, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey,
+                     result.sessionToken, result.region, result.backupKey ];
+
+        async.series([
+            addons.backupAddons.bind(null, app, addonsToBackup),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
+        ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            debugApp(app, 'createNewAppBackup: %s done', result.id);
+
+            backupdb.add({ id: result.id, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, result.id);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof lastBackupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function backupApp(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appConfig = null, backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        appConfig = app.lastBackupConfig;
+        backupFunction = reuseOldAppBackup.bind(null, app);
+    } else {
+        appConfig = {
+            manifest: app.manifest,
+            location: app.location,
+            portBindings: app.portBindings,
+            accessRestriction: app.accessRestriction,
+            memoryLimit: app.memoryLimit
+        };
+        backupFunction = createNewAppBackup.bind(null, app, addonsToBackup);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(error);
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, appConfig, function (error) {
+            if (error) return callback(error);
+
+            return callback(null, backupId);
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            backupApp(app, app.manifest.addons, function (error, backupId) {
+                if (error && error.reason !== BackupsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
+                }
+
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+
+                callback(error, filename);
+            });
+        });
+    });
+}
+
+function backup(eventSource, callback) {
+    assert.strictEqual(typeof eventSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
+
+    eventlog.add(eventlog.ACTION_BACKUP_START, eventSource, { });
+
+    progress.set(progress.BACKUP, 0, 'Starting'); // ensure tools can 'wait' on progress
+
+    backupBoxAndApps(function (error, filename) { // start the backup operation in the background
+        if (error) console.error('backup failed.', error);
+
+        eventlog.add(eventlog.ACTION_BACKUP_FINISH, eventSource, { errorMessage: error ? error.message : null, filename: filename });
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    getPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        var eventSource = { userId: null, username: 'cron' };
+        backup(eventSource, callback);
+    });
+}
+
+function restoreApp(app, addonsToRestore, backupId, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    getRestoreUrl(backupId, function (error, result) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
+            if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
+        });
+    });
+}

src/certificates.js

@@ -9,6 +9,7 @@ var acme = require('./cert/acme.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:src/certificates'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     mailer = require('./mailer.js'),
     nginx = require('./nginx.js'),
@@ -17,7 +18,6 @@ var acme = require('./cert/acme.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
-    tld = require('tldjs'),
     user = require('./user.js'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
@@ -57,11 +57,14 @@ util.inherits(CertificatesError, Error);
 CertificatesError.INTERNAL_ERROR = 'Internal Error';
 CertificatesError.INVALID_CERT = 'Invalid certificate';
 
-function getApi(callback) {
+function getApi(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     settings.getTlsConfig(function (error, tlsConfig) {
         if (error) return callback(error);
 
-        var api = tlsConfig.provider === 'caas' ? caas : acme;
+        var api = !app.altDomain && tlsConfig.provider === 'caas' ? caas : acme;
 
         var options = { };
         options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
@@ -89,11 +92,10 @@ function installAdminCertificate(callback) {
         sysinfo.getIp(function (error, ip) {
             if (error) return callback(error);
 
-            var zoneName = tld.getDomain(config.fqdn());
-            waitForDns(config.adminFqdn(), ip, zoneName, function (error) {
+            waitForDns(config.adminFqdn(), ip, 'A', function (error) {
                 if (error) return callback(error); // this cannot happen because we retry forever
 
-                ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
+                ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
                     if (error) { // currently, this can never happen
                         debug('Error obtaining certificate. Proceed anyway', error);
                         return callback();
@@ -126,11 +128,11 @@ function autoRenew(callback) {
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        allApps.push({ location: 'my' }); // inject fake webadmin app
+        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app
 
         var expiringApps = [ ];
         for (var i = 0; i < allApps.length; i++) {
-            var appDomain = config.appFqdn(allApps[i].location);
+            var appDomain = allApps[i].altDomain || config.appFqdn(allApps[i].location);
             var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
             var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
@@ -144,20 +146,23 @@ function autoRenew(callback) {
             }
         }
 
-        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return config.appFqdn(a.location); }));
+        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
 
-        getApi(function (error, api, apiOptions) {
-            if (error) return callback(error);
+        async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+            var domain = app.altDomain || config.appFqdn(app.location);
+
+            getApi(app, function (error, api, apiOptions) {
+                if (error) return callback(error);
 
-            async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
-                var domain = config.appFqdn(app.location);
                 debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
 
                 api.getCertificate(domain, apiOptions, function (error) {
                     var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
                     var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
 
-                    mailer.certificateRenewed(domain, error ? error.message : '');
+                    var errorMessage = error ? error.message : '';
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, { userId: null, username: 'cron' }, { domain: domain, errorMessage: errorMessage });
+                    mailer.certificateRenewed(domain, errorMessage);
 
                     if (error) {
                         debug('autoRenew: could not renew cert for %s because %s', domain, error);
@@ -257,7 +262,7 @@ function setAdminCertificate(cert, key, callback) {
     assert.strictEqual(typeof key, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
+    var vhost = config.adminFqdn();
     var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
     var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
 
@@ -271,10 +276,12 @@ function setAdminCertificate(cert, key, callback) {
     nginx.configureAdmin(certFilePath, keyFilePath, callback);
 }
 
-function ensureCertificate(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
+function ensureCertificate(app, callback) {
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    var domain = app.altDomain || config.appFqdn(app.location);
+
     // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
     var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
     var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
@@ -287,7 +294,7 @@ function ensureCertificate(domain, callback) {
 
     debug('ensureCertificate: %s cert require renewal', domain);
 
-    getApi(function (error, api, apiOptions) {
+    getApi(app, function (error, api, apiOptions) {
         if (error) return callback(error);
 
         debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);

src/clients.js

@@ -8,7 +8,14 @@ exports = module.exports = {
     del: del,
     getAllWithDetailsByUserId: getAllWithDetailsByUserId,
     getClientTokensByUserId: getClientTokensByUserId,
-    delClientTokensByUserId: delClientTokensByUserId
+    delClientTokensByUserId: delClientTokensByUserId,
+
+    SCOPE_APPS: 'apps',
+    SCOPE_DEVELOPER: 'developer',
+    SCOPE_PROFILE: 'profile',
+    SCOPE_ROOT: 'root',
+    SCOPE_SETTINGS: 'settings',
+    SCOPE_USERS: 'users'
 };
 
 var assert = require('assert'),
@@ -47,10 +54,20 @@ ClientsError.INVALID_CLIENT = 'Invalid client';
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
 
+    var VALID_SCOPES = [
+        exports.SCOPE_APPS,
+        exports.SCOPE_DEVELOPER,
+        exports.SCOPE_PROFILE,
+        exports.SCOPE_ROOT,
+        exports.SCOPE_SETTINGS,
+        exports.SCOPE_USERS
+    ];
+
     if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
     if (scope === '*') return null;
 
-    // TODO maybe validate all individual scopes if they exist
+    var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE);
 
     return null;
 }

src/cloudron.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -16,9 +14,7 @@ exports = module.exports = {
     updateToLatest: updateToLatest,
     update: update,
     reboot: reboot,
-    backup: backup,
     retire: retire,
-    ensureBackup: ensureBackup,
 
     isConfiguredSync: isConfiguredSync,
 
@@ -27,19 +23,19 @@ exports = module.exports = {
     events: new (require('events').EventEmitter)(),
 
     EVENT_ACTIVATED: 'activated',
-    EVENT_CONFIGURED: 'configured'
+    EVENT_CONFIGURED: 'configured',
+    EVENT_FIRST_RUN: 'firstrun'
 };
 
 var apps = require('./apps.js'),
-    AppsError = require('./apps.js').AppsError,
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
     debug = require('debug')('box:cloudron'),
     df = require('node-df'),
+    eventlog = require('./eventlog.js'),
     fs = require('fs'),
     locker = require('./locker.js'),
     mailer = require('./mailer.js'),
@@ -59,11 +55,9 @@ var apps = require('./apps.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
     util = require('util'),
-    webhooks = require('./webhooks.js');
+    uuid = require('node-uuid');
 
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
     RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');
 
@@ -71,24 +65,9 @@ var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
 var gUpdatingDns = false,                // flag for dns update reentrancy
     gCloudronDetails = null,             // cached cloudron details like region,size...
+    gAppstoreUserDetails = {},
     gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
 
-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
-
 function CloudronError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -124,14 +103,33 @@ function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.on(exports.EVENT_FIRST_RUN, installAppBundle);
 
-    syncConfigState(callback);
+    // check activation state for existing cloudrons that do not have first run file
+    // can be removed once cloudrons have been updated
+    isActivated(function (error, activated) {
+        if (error) return callback(error);
+
+        debug('initialize: cloudron %s activated', activated ? '' : 'not');
+
+        if (activated) fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+
+        if (!fs.existsSync(paths.FIRST_RUN_FILE)) {
+            // EE API is sync. do not keep the server waiting
+            debug('initialize: emitting first run event');
+            process.nextTick(function () { exports.events.emit(exports.EVENT_FIRST_RUN); });
+            fs.writeFileSync(paths.FIRST_RUN_FILE, 'been there, done that', 'utf8');
+        }
+
+        syncConfigState(callback);
+    });
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
+    exports.events.removeListener(exports.EVENT_FIRST_RUN, installAppBundle);
 
     callback(null);
 }
@@ -140,6 +138,15 @@ function isConfiguredSync() {
     return gIsConfigured === true;
 }
 
+function isActivated(callback) {
+    user.getOwner(function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+        if (error) return callback(error);
+
+        callback(null, true);
+    });
+}
+
 function isConfigured(callback) {
     // set of rules to see if we have the configs required for cloudron to function
     // note this checks for missing configs and not invalid configs
@@ -201,19 +208,20 @@ function setTimeZone(ip, callback) {
     });
 }
 
-function activate(username, password, email, displayName, ip, callback) {
+function activate(username, password, email, displayName, ip, auditSource, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof displayName, 'string');
     assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('activating user:%s email:%s', username, email);
 
     setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
-    user.createOwner(username, password, email, displayName, function (error, userObject) {
+    user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
         if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
         if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
@@ -233,6 +241,8 @@ function activate(username, password, email, displayName, ip, callback) {
                 // EE API is sync. do not keep the REST API reponse waiting
                 process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });
 
+                eventlog.add(eventlog.ACTION_ACTIVATE, auditSource, { });
+
                 callback(null, { token: token, expires: expires });
             });
         });
@@ -282,6 +292,7 @@ function getCloudronDetails(callback) {
             if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
 
             gCloudronDetails = result.body.box;
+            gAppstoreUserDetails = result.body.user;
 
             return callback(null, gCloudronDetails);
         });
@@ -323,6 +334,7 @@ function getConfig(callback) {
                         developerMode: developerMode,
                         region: result.region,
                         size: result.size,
+                        billing: !!gAppstoreUserDetails.billing,
                         memory: os.totalmem(),
                         provider: config.provider(),
                         cloudronName: cloudronName
@@ -502,12 +514,15 @@ function update(boxUpdateInfo, callback) {
 }
 
 
-function updateToLatest(callback) {
+function updateToLatest(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var boxUpdateInfo = updateChecker.getUpdateInfo().box;
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
 
+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
     update(boxUpdateInfo, callback);
 }
 
@@ -531,7 +546,7 @@ function doUpgrade(boxUpdateInfo, callback) {
 
     progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
 
-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps(function (error) {
         if (error) return upgradeError(error);
 
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
@@ -560,7 +575,7 @@ function doUpdate(boxUpdateInfo, callback) {
 
     progress.set(progress.UPDATE, 5, 'Backing up for update');
 
-    backupBoxAndApps(function (error) {
+    backups.backupBoxAndApps(function (error) {
         if (error) return updateError(error);
 
         // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
@@ -607,120 +622,38 @@ function doUpdate(boxUpdateInfo, callback) {
     });
 }
 
-function backup(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_FULL_BACKUP);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    // ensure tools can 'wait' on progress
-    progress.set(progress.BACKUP, 0, 'Starting');
-
-    // start the backup operation in the background
-    backupBoxAndApps(function (error) {
-        if (error) console.error('backup failed.', error);
-
-        locker.unlock(locker.OP_FULL_BACKUP);
-    });
-
-    callback(null);
-}
-
-function ensureBackup(callback) {
+function installAppBundle(callback) {
     callback = callback || NOOP_CALLBACK;
 
-    backups.getPaged(1, 1, function (error, backups) {
-        if (error) {
-            debug('Unable to list backups', error);
-            return callback(error); // no point trying to backup if appstore is down
-        }
+    var bundle = config.get('appBundle');
 
-        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
-            debug('Previous backup was %j, no need to backup now', backups[0]);
-            return callback(null);
-        }
+    if (!bundle || bundle.length === 0) {
+        debug('installAppBundle: no bundle set');
+        return callback();
+    }
 
-        backup(callback);
-    });
-}
+    async.eachSeries(bundle, function (appInfo, iteratorCallback) {
+        var appstoreId = appInfo.appstoreId;
+        var parts = appstoreId.split('@');
 
-function backupBoxWithAppBackupIds(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
+        var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
 
-    backups.getBackupUrl(appBackupIds, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+        superagent.get(url).end(function (error, result) {
+            if (error && !error.response) return iteratorCallback(new Error('Network error: ' + error.message));
 
-        debug('backup: url %s', result.url);
+            if (result.statusCode !== 200) return iteratorCallback(util.format('Failed to get app info from store.', result.statusCode, result.text));
 
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+            debug('autoInstall: installing %s at %s', appstoreId, appInfo.location);
 
-            debug('backup: successful');
-
-            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                if (error) return callback(error);
-                callback(null, result.id);
-            });
+            apps.install(uuid.v4(), appstoreId, result.body.manifest, appInfo.location,
+                appInfo.portBindings || null, appInfo.accessRestriction || null,
+                null /* icon */, null /* cert */, null /* key */, 0 /* default mem limit */,
+                null /* altDomain */, { userId: null, username: 'autoinstaller' }, iteratorCallback);
         });
-    });
-}
+    }, function (error) {
+        if (error) debug('autoInstallApps: ', error);
 
-// this function expects you to have a lock
-function backupBox(callback) {
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
-
-        backupBoxWithAppBackupIds(appBackupIds, callback);
-    });
-}
-
-// this function expects you to have a lock
-function backupBoxAndApps(callback) {
-    callback = callback || NOOP_CALLBACK;
-
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var processed = 0;
-        var step = 100/(allApps.length+1);
-
-        progress.set(progress.BACKUP, processed, '');
-
-        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
-            ++processed;
-
-            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                if (error && error.reason !== AppsError.BAD_STATE) {
-                    debugApp(app, 'Unable to backup', error);
-                    return iteratorCallback(error);
-                }
-
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
-                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
-            });
-        }, function appsBackedUp(error, backupIds) {
-            if (error) {
-                progress.set(progress.BACKUP, 100, error.message);
-                return callback(error);
-            }
-
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
-
-            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
-                progress.set(progress.BACKUP, 100, error ? error.message : '');
-
-                callback(error, filename);
-            });
-        });
+        callback();
     });
 }
 

src/collectd.config.ejs

@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.stat">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
     Instance "<%= appId %>-memory"
     Separator " \\n"
     <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
     Instance "<%= appId %>-memory"
     Separator "\\n"
     <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker/<%= containerId %>/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
     Instance "<%= appId %>-cpu"
     Separator " \\n"
     <Result>

src/config.js

@@ -1,11 +1,7 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
     baseDir: baseDir,
-    dnsInSync: dnsInSync,
-    setDnsInSync: setDnsInSync,
 
     // values set here will be lost after a upgrade/update. use the sqlite database
     // for persistent values that need to be backed up
@@ -30,6 +26,7 @@ exports = module.exports = {
     // these values are derived
     adminOrigin: adminOrigin,
     internalAdminOrigin: internalAdminOrigin,
+    sysadminOrigin: sysadminOrigin, // caas routes
     adminFqdn: adminFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
@@ -59,14 +56,6 @@ function baseDir() {
 
 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
 
-function dnsInSync() {
-    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
-}
-
-function setDnsInSync(content) {
-    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
-}
-
 function saveSync() {
     fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
@@ -89,11 +78,12 @@ function initConfig() {
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
-    data.internalPort = 3001;
+    data.sysadminPort = 3001;
     data.ldapPort = 3002;
     data.oauthProxyPort = 3003;
     data.simpleAuthPort = 3004;
     data.provider = 'caas';
+    data.appBundle = [ ];
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -150,7 +140,7 @@ function get(key) {
 }
 
 function adminEmail() {
-    return '"Cloudron" ' + get('adminEmail');
+    return get('adminEmail');
 }
 
 function apiServerOrigin() {
@@ -185,6 +175,10 @@ function internalAdminOrigin() {
     return 'http://127.0.0.1:' + get('port');
 }
 
+function sysadminOrigin() {
+    return 'http://127.0.0.1:' + get('sysadminPort');
+}
+
 function token() {
     return get('token');
 }

src/cron.js

@@ -7,6 +7,7 @@ exports = module.exports = {
 
 var apps = require('./apps.js'),
     assert = require('assert'),
+    backups = require('./backups.js'),
     certificates = require('./certificates.js'),
     cloudron = require('./cloudron.js'),
     config = require('./config.js'),
@@ -65,7 +66,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
             cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: cloudron.ensureBackup,
+            onTick: backups.ensureBackup,
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });

src/database.js

@@ -116,11 +116,13 @@ function clear(callback) {
     async.series([
         require('./appdb.js')._clear,
         require('./authcodedb.js')._clear,
+        require('./backupdb.js')._clear,
         require('./clientdb.js')._clear,
         require('./tokendb.js')._clear,
         require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear
+        require('./settingsdb.js')._clear,
+        require('./eventlogdb.js')._clear
     ], callback);
 }
 

src/developer.js

@@ -14,6 +14,7 @@ exports = module.exports = {
 var assert = require('assert'),
     config = require('./config.js'),
     debug = require('debug')('box:developer'),
+    eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
     settings = require('./settings.js'),
     superagent = require('superagent'),
@@ -50,26 +51,33 @@ function enabled(callback) {
     });
 }
 
-function setEnabled(enabled, callback) {
+function setEnabled(enabled, auditSource, callback) {
     assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     settings.setDeveloperMode(enabled, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        eventlog.add(eventlog.ACTION_CLI_MODE, auditSource, { enabled: enabled });
+
         callback(null);
     });
 }
 
-function issueDeveloperToken(user, callback) {
+function issueDeveloperToken(user, auditSource, callback) {
     assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
+    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users,profile', function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
+        eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
+
         callback(null, { token: token, expiresAt: expiresAt });
     });
 }

src/docker.js

@@ -1,17 +1,5 @@
 'use strict';
 
-var addons = require('./addons.js'),
-    async = require('async'),
-    assert = require('assert'),
-    config = require('./config.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:src/docker.js'),
-    Docker = require('dockerode'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    util = require('util'),
-    _ = require('underscore');
-
 exports = module.exports = {
     connection: connectionInstance(),
     downloadImage: downloadImage,
@@ -25,10 +13,12 @@ exports = module.exports = {
     deleteImage: deleteImage,
     deleteContainers: deleteContainers,
     createSubcontainer: createSubcontainer,
-    getContainerIdByIp: getContainerIdByIp
+    getContainerIdByIp: getContainerIdByIp,
+    execContainer: execContainer
 };
 
 function connectionInstance() {
+    var Docker = require('dockerode');
     var docker;
 
     if (process.env.BOX_ENV === 'test') {
@@ -44,6 +34,20 @@ function connectionInstance() {
     return docker;
 }
 
+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    child_process = require('child_process'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/docker.js'),
+    once = require('once'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    spawn = child_process.spawn,
+    util = require('util'),
+    _ = require('underscore');
+
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
 
@@ -134,12 +138,13 @@ function createSubcontainer(app, name, cmd, options, callback) {
     var manifest = app.manifest;
     var developmentMode = !!manifest.developmentMode;
     var exposedPorts = {}, dockerPortBindings = { };
+    var domain = app.altDomain || config.appFqdn(app.location);
     var stdEnv = [
         'CLOUDRON=1',
         'WEBADMIN_ORIGIN=' + config.adminOrigin(),
         'API_ORIGIN=' + config.adminOrigin(),
-        'APP_ORIGIN=https://' + config.appFqdn(app.location),
-        'APP_DOMAIN=' + config.appFqdn(app.location)
+        'APP_ORIGIN=https://' + domain,
+        'APP_DOMAIN=' + domain
     ];
 
     // docker portBindings requires ports to be exposed
@@ -389,3 +394,34 @@ function getContainerIdByIp(ip, callback) {
         callback(null, containerId);
     });
 }
+
+function execContainer(containerId, cmd, options, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert(util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', '-i', containerId  ].concat(cmd));
+
+    var chunks = [ ];
+
+    if (options.stdout) {
+        cp.stdout.pipe(options.stdout);
+    } else if (options.bufferStdout) {
+        cp.stdout.on('data', function (chunk) { chunks.push(chunk); });
+    } else {
+        cp.stdout.pipe(process.stdout);
+    }
+
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debug('execContainer code: %s signal: %s', code, signal);
+        if (!callback.called) callback(code ? 'Failed with status ' + code : null, Buffer.concat(chunks));
+    });
+
+    cp.stderr.pipe(options.stderr || process.stderr);
+
+    if (options.stdin) options.stdin.pipe(cp.stdin).on('error', callback);
+}

src/eventlog.js

@@ -0,0 +1,99 @@
+'use strict';
+
+exports = module.exports = {
+    EventLogError: EventLogError,
+
+    add: add,
+    get: get,
+    getAllPaged: getAllPaged,
+
+    // keep in sync with webadmin index.js filter
+    ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CONFIGURE: 'app.configure',
+    ACTION_APP_INSTALL: 'app.install',
+    ACTION_APP_RESTORE: 'app.restore',
+    ACTION_APP_UNINSTALL: 'app.uninstall',
+    ACTION_APP_UPDATE: 'app.update',
+    ACTION_BACKUP_FINISH: 'backup.finish',
+    ACTION_BACKUP_START: 'backup.start',
+    ACTION_CERTIFICATE_RENEWAL: 'certificate.renew',
+    ACTION_CLI_MODE: 'settings.climode',
+    ACTION_START: 'cloudron.start',
+    ACTION_UPDATE: 'cloudron.update',
+    ACTION_USER_ADD: 'user.add',
+    ACTION_USER_LOGIN: 'user.login',
+    ACTION_USER_REMOVE: 'user.remove',
+    ACTION_USER_UPDATE: 'user.update'
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:eventlog'),
+    eventlogdb = require('./eventlogdb.js'),
+    util = require('util'),
+    uuid = require('node-uuid');
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function EventLogError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(EventLogError, Error);
+EventLogError.INTERNAL_ERROR = 'Internal error';
+EventLogError.NOT_FOUND = 'Not Found';
+
+function add(action, source, data, callback) {
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    var id = uuid.v4();
+
+    eventlogdb.add(id, action, source, data, function (error) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, { id: id });
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new EventLogError(EventLogError.NOT_FOUND, 'No such event'));
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAllPaged(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    eventlogdb.getAllPaged(page, perPage, function (error, boxes) {
+        if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
+
+        callback(null, boxes);
+    });
+}

src/eventlogdb.js

@@ -0,0 +1,84 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getAllPaged: getAllPaged,
+    add: add,
+    count: count,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    safe = require('safetydance');
+
+var EVENTLOGS_FIELDS = [ 'id', 'action', 'source', 'data', 'creationTime' ].join(',');
+
+// until mysql module supports automatic type coercion
+function postProcess(eventLog) {
+    eventLog.source = safe.JSON.parse(eventLog.source);
+    eventLog.data = safe.JSON.parse(eventLog.data);
+    return eventLog;
+}
+
+function get(eventId, callback) {
+    assert.strictEqual(typeof eventId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE id = ?', [ eventId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, postProcess(result[0]));
+    });
+}
+
+function getAllPaged(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog ORDER BY creationTime DESC LIMIT ?,?', [ (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, action, source, data, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof action, 'string');
+    assert.strictEqual(typeof source, 'object');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO eventlog (id, action, source, data) VALUES (?, ?, ?, ?)', [ id, action, JSON.stringify(source), JSON.stringify(data) ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM eventlog', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM eventlog', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(error);
+    });
+}
+

src/ldap.js

@@ -9,6 +9,7 @@ var assert = require('assert'),
     apps = require('./apps.js'),
     config = require('./config.js'),
     debug = require('debug')('box:ldap'),
+    eventlog = require('./eventlog.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     ldap = require('ldapjs');
@@ -57,7 +58,12 @@ function start(callback) {
                 var groups = [ GROUP_USERS_DN ];
                 if (entry.admin) groups.push(GROUP_ADMINS_DN);
 
-                var tmp = {
+                var displayName = entry.displayName || entry.username;
+                var nameParts = displayName.split(' ');
+                var firstName = nameParts[0];
+                var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+                var obj = {
                     dn: dn.toString(),
                     attributes: {
                         objectclass: ['user'],
@@ -65,15 +71,23 @@ function start(callback) {
                         cn: entry.id,
                         uid: entry.id,
                         mail: entry.email,
-                        displayname: entry.displayName || entry.username,
+                        displayname: displayName,
+                        givenName: firstName,
                         username: entry.username,
                         samaccountname: entry.username,      // to support ActiveDirectory clients
                         memberof: groups
                     }
                 };
 
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
+                // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+                // which is required to have atleast one character if present
+                if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+                // ensure all filter values are also lowercase
+                var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                    res.send(obj);
                 }
             });
 
@@ -99,7 +113,7 @@ function start(callback) {
                 var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
                 var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
 
-                var tmp = {
+                var obj = {
                     dn: dn.toString(),
                     attributes: {
                         objectclass: ['group'],
@@ -108,8 +122,11 @@ function start(callback) {
                     }
                 };
 
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                    res.send(tmp);
+                // ensure all filter values are also lowercase
+                var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                    res.send(obj);
                 }
             });
 
@@ -131,8 +148,15 @@ function start(callback) {
         var commonName = req.dn.rdns[0][attributeName];
         if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-        // if mail is specified, enforce mail check, otherwise allow both
-        var api = (commonName.indexOf('@') === -1) && (attributeName !== 'mail')  ? user.verify : user.verifyWithEmail;
+        var api;
+        // if mail is specified, enforce mail check
+        if (commonName.indexOf('@') !== -1 || attributeName === 'mail') {
+            api = user.verifyWithEmail;
+        } else if (commonName.indexOf('uid-') === 0) {
+            api = user.verify;
+        } else {
+            api = user.verifyWithUsername;
+        }
 
         // TODO this should be done after we verified the app has access to avoid leakage of user existence
         api(commonName, req.credentials || '', function (error, userObject) {
@@ -154,6 +178,8 @@ function start(callback) {
                     // we return no such object, to avoid leakage of a users existence
                     if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
+                    eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: userObject.id });
+
                     res.end();
                 });
             });

src/logcollector.js

@@ -0,0 +1,37 @@
+'use strict';
+
+exports = module.exports = {
+    sendFailureLogs: sendFailureLogs
+};
+
+var assert = require('assert'),
+    mailer = require('./mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
+
+function collectLogs(unitName, callback) {
+    assert.strictEqual(typeof unitName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendFailureLogs(processName, options) {
+    assert.strictEqual(typeof processName, 'string');
+    assert.strictEqual(typeof options, 'object');
+
+    collectLogs(options.unit || processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending failure logs for', processName);
+
+        mailer.unexpectedExit(processName, result);
+    });
+}

src/mail_templates/password_reset.ejs

@@ -1,6 +1,6 @@
 <%if (format === 'text') { %>
 
-Dear <%= username %>,
+Dear <%= user.username || user.email %>,
 
 Someone, hopefully you, has requested your <%= fqdn %>'s account password
 be reset. If you did not request this reset, please ignore this message.

src/mail_templates/unexpected_exit.ejs

@@ -2,7 +2,7 @@
 
 Dear Cloudron Team,
 
-Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> exited unexpectedly!
 
 Please see some excerpt of the logs below.
 

src/mail_templates/user_added.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-User with name '<%= username %>' (<%= email %>) was added in the Cloudron at <%= fqdn %>.
+User with name <%= user.email %> was added in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mail_templates/user_event.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-User with name '<%= username %>' (<%= email %>) <%= event %> in the Cloudron at <%= fqdn %>.
+User <%= user.username %> <%= user.email %> <%= event %> in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mail_templates/welcome_user.ejs

@@ -1,20 +1,15 @@
 <%if (format === 'text') { %>
 
-Dear <%= user.username %>,
+Dear <%= user.email %>,
 
 Welcome to our Cloudron <%= fqdn %>!
 
 The Cloudron is our own Smart Server. You can read more about it
 at https://www.cloudron.io.
 
-           You username is '<%= user.username %>'
-
 To get started, create your account by visiting the following page:
 <%= setupLink %>
 
-When you visit the above page, you will be prompted to enter a new password.
-After you have submitted the form, you can login using the new password.
-
 <% if (invitor && invitor.email) { %>
 Thank you,
 <%= invitor.email %>

src/mailer.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -14,7 +12,7 @@ exports = module.exports = {
     appUpdateAvailable: appUpdateAvailable,
 
     sendInvite: sendInvite,
-    sendCrashNotification: sendCrashNotification,
+    unexpectedExit: unexpectedExit,
 
     appDied: appDied,
 
@@ -26,6 +24,7 @@ exports = module.exports = {
     FEEDBACK_TYPE_TICKET: 'ticket',
     FEEDBACK_TYPE_APP_MISSING: 'app_missing',
     FEEDBACK_TYPE_APP_ERROR: 'app_error',
+    FEEDBACK_TYPE_UPGRADE_REQUEST: 'upgrade_request',
     sendFeedback: sendFeedback,
 
     _getMailQueue: _getMailQueue,
@@ -200,6 +199,8 @@ function getAdminEmails(callback) {
     users.getAllAdmins(function (error, admins) {
         if (error) return callback(error);
 
+        if (admins.length === 0) return callback(new Error('No admins on this cloudron')); // box not activated yet
+
         var adminEmails = [ ];
         admins.forEach(function (admin) { adminEmails.push(admin.email); });
 
@@ -219,8 +220,8 @@ function mailUserEventToAdmins(user, event) {
         var mailOptions = {
             from: config.adminEmail(),
             to: adminEmails.join(', '),
-            subject: util.format('%s %s in Cloudron %s', user.username, event, config.fqdn()),
-            text: render('user_event.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, event: event, format: 'text' }),
+            subject: util.format('%s %s in Cloudron %s', user.username || user.email, event, config.fqdn()),
+            text: render('user_event.ejs', { fqdn: config.fqdn(), user: user, event: event, format: 'text' }),
         };
 
         enqueue(mailOptions);
@@ -236,7 +237,7 @@ function sendInvite(user, invitor) {
     var templateData = {
         user: user,
         webadminUrl: config.adminOrigin(),
-        setupLink: config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken,
+        setupLink: config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken,
         format: 'text',
         fqdn: config.fqdn(),
         invitor: invitor
@@ -263,25 +264,25 @@ function userAdded(user, inviteSent) {
 
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
-        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken;
+        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken;
 
         var mailOptions = {
             from: config.adminEmail(),
             to: adminEmails.join(', '),
-            subject: util.format('%s added in Cloudron %s', user.username, config.fqdn()),
-            text: render('user_added.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, inviteLink: inviteLink, format: 'text' }),
+            subject: util.format('%s added in Cloudron %s', user.email, config.fqdn()),
+            text: render('user_added.ejs', { fqdn: config.fqdn(), user: user, inviteLink: inviteLink, format: 'text' }),
         };
 
         enqueue(mailOptions);
     });
 }
 
-function userRemoved(username) {
-    assert.strictEqual(typeof username, 'string');
+function userRemoved(user) {
+    assert.strictEqual(typeof user, 'object');
 
-    debug('Sending mail for userRemoved');
+    debug('Sending mail for userRemoved.', user.id, user.email);
 
-    mailUserEventToAdmins({ username: username }, 'was removed');
+    mailUserEventToAdmins(user, 'was removed');
 }
 
 function adminChanged(user, admin) {
@@ -296,7 +297,7 @@ function adminChanged(user, admin) {
 function passwordReset(user) {
     assert.strictEqual(typeof user, 'object');
 
-    debug('Sending mail for password reset for user %s.', user.username);
+    debug('Sending mail for password reset for user %s.', user.email, user.id);
 
     var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
 
@@ -304,7 +305,7 @@ function passwordReset(user) {
         from: config.adminEmail(),
         to: user.email,
         subject: 'Password Reset Request',
-        text: render('password_reset.ejs', { fqdn: config.fqdn(), username: user.username, resetLink: resetLink, format: 'text' })
+        text: render('password_reset.ejs', { fqdn: config.fqdn(), user: user, resetLink: resetLink, format: 'text' })
     };
 
     enqueue(mailOptions);
@@ -394,7 +395,7 @@ function certificateRenewed(domain, message) {
 
 // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
 // crashnotifier should be able to send mail when there is no db
-function sendCrashNotification(program, context) {
+function unexpectedExit(program, context) {
     assert.strictEqual(typeof program, 'string');
     assert.strictEqual(typeof context, 'string');
 
@@ -402,7 +403,7 @@ function sendCrashNotification(program, context) {
         from: config.adminEmail(),
         to: 'admin@cloudron.io',
         subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
-        text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
+        text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
     };
 
     sendMails([ mailOptions ]);
@@ -417,6 +418,7 @@ function sendFeedback(user, type, subject, description) {
     assert(type === exports.FEEDBACK_TYPE_TICKET ||
         type === exports.FEEDBACK_TYPE_FEEDBACK ||
         type === exports.FEEDBACK_TYPE_APP_MISSING ||
+        type === exports.FEEDBACK_TYPE_UPGRADE_REQUEST ||
         type === exports.FEEDBACK_TYPE_APP_ERROR);
 
     var mailOptions = {

src/nginx.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var assert = require('assert'),
@@ -66,7 +64,7 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
     var sourceDir = path.resolve(__dirname, '..');
     var oauthProxy = requiresOAuthProxy(app);
     var endpoint = oauthProxy ? 'oauthproxy' : 'app';
-    var vhost = config.appFqdn(app.location);
+    var vhost = app.altDomain || config.appFqdn(app.location);
 
     var data = {
         sourceDir: sourceDir,
@@ -80,10 +78,10 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
     var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-    debug('writing config for "%s" to %s', app.location, nginxConfigFilename);
+    debug('writing config for "%s" to %s', vhost, nginxConfigFilename);
 
     if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-        debug('Error creating nginx config for "%s" : %s', app.location, safe.error.message);
+        debug('Error creating nginx config for "%s" : %s', vhost, safe.error.message);
         return callback(safe.error);
     }
 
@@ -94,9 +92,11 @@ function unconfigureApp(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    var vhost = app.altDomain || config.appFqdn(app.location);
+
     var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
     if (!safe.fs.unlinkSync(nginxConfigFilename)) {
-        debug('Error removing nginx configuration of "%s": %s', app.location, safe.error.message);
+        debug('Error removing nginx configuration of "%s": %s', vhost, safe.error.message);
         return callback(null);
     }
 

src/oauth2views/account_setup.ejs

@@ -0,0 +1,68 @@
+<% include header %>
+
+<!-- tester -->
+
+<script>
+
+'use strict';
+
+// very basic angular app
+var app = angular.module('Application', []);
+app.controller('Controller', [function () {}]);
+
+</script>
+
+<center>
+    <br/>
+    <h4>Hello <%= (user && user.email) ? user.email : '' %>, welcome to your Cloudron.</h4>
+    <h2>Setup your account and password.</h2>
+</center>
+
+<div class="container" ng-app="Application" ng-controller="Controller">
+    <div class="row">
+        <div class="col-md-6 col-md-offset-3">
+            <form action="/api/v1/session/account/setup" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
+                <input type="password" style="display: none;">
+                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
+
+                <center><p class="has-error"><%= error %></p></center>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
+                    <label class="control-label">Username</label>
+                    <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
+                        <small ng-show="setupForm.username.$error.minlength">The username is too short</small>
+                        <small ng-show="setupForm.username.$error.maxlength">The username is too long</small>
+                        <small ng-show="setupForm.username.$dirty && setupForm.username.$invalid">Not a valid username</small>
+                    </div>
+                    <input type="text" class="form-control" ng-model="username" name="username" ng-maxlength="512" ng-minlength="3" required autofocus>
+                </div>
+
+                <div class="form-group">
+                    <label class="control-label">Display Name</label>
+                    <input type="displayName" class="form-control" ng-model="displayName" name="displayName" required>
+                </div>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.password.$dirty && setupForm.password.$invalid) }">
+                    <label class="control-label">New Password</label>
+                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
+                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+                    </div>
+                    <input type="password" class="form-control" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" required>
+                </div>
+
+                <div class="form-group" ng-class="{ 'has-error': (setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)) }">
+                    <label class="control-label">Repeat Password</label>
+                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
+                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
+                    </div>
+                    <input type="password" class="form-control" ng-model="passwordRepeat" name="passwordRepeat" required>
+                </div>
+
+                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
+            </form>
+        </div>
+    </div>
+</div>
+
+<% include footer %>

src/oauth2views/header.ejs

@@ -8,9 +8,11 @@
 
     <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
 
+    <!-- Theme CSS -->
+    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
-    <link href="https://fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
     <!-- jQuery-->
     <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
@@ -22,9 +24,6 @@
     <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
     <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
-
 </head>
 
 <body class="oauth">

src/oauth2views/password_reset.ejs

@@ -13,13 +13,14 @@ app.controller('Controller', [function () {}]);
 </script>
 
 <center>
-    <h1>Hello <%= user.username %> create a new password</h1>
+    <h1>Hello <%= user.username %>, set a new password</h1>
 </center>
 
 <div class="container" ng-app="Application" ng-controller="Controller">
     <div class="row">
         <div class="col-md-6 col-md-offset-3">
             <form action="/api/v1/session/password/reset" method="post" name="resetForm" autocomplete="off" role="form" novalidate>
+                <input type="password" style="display: none;">
                 <input type="hidden" name="_csrf" value="<%= csrf %>"/>
                 <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
 

src/oauth2views/password_reset_sent.ejs

@@ -3,14 +3,14 @@
 <!-- tester -->
 
 <center>
-    <h1>Reset your password successful</h1>
+    <h1>Password reset successful</h1>
 </center>
 
 <div class="container">
     <div class="row">
         <div class="col-md-6 col-md-offset-3">
-            <p>An email was sent to you with a link to create a new password.</p>
-            If you have not received any email after some time, maybe you have misspelled your email address, simply try again <a href="/api/v1/session/password/resetRequest.html">here</a>.
+            <p>An email was sent to you with a link to set a new password.</p>
+            If you have not received any email, simply <a href="/api/v1/session/password/resetRequest.html">try again</a>.
         </div>
     </div>
 </div>

src/oauth2views/password_setup.ejs

@@ -1,46 +0,0 @@
-<% include header %>
-
-<!-- tester -->
-
-<script>
-
-'use strict';
-
-// very basic angular app
-var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
-
-</script>
-
-<center>
-    <h1>Hello <%= user.username %>, set a password</h1>
-</center>
-
-<div class="container" ng-app="Application" ng-controller="Controller">
-    <div class="row">
-        <div class="col-md-6 col-md-offset-3">
-            <form action="/api/v1/session/password/reset" method="post" name="setupForm" autocomplete="off" role="form" novalidate>
-                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-                <input type="hidden" name="resetToken" value="<%= resetToken %>"/>
-
-                <div class="form-group" ng-class="{ 'has-error': setupForm.password.$dirty && setupForm.password.$invalid }">
-                    <label class="control-label" for="inputPassword">New Password</label>
-                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
-                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
-                    </div>
-                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" autofocus required>
-                </div>
-                <div class="form-group" ng-class="{ 'has-error': setupForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
-                    <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
-                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
-                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
-                    </div>
-                    <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
-                </div>
-                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>
-            </form>
-        </div>
-    </div>
-</div>
-
-<% include footer %>

src/paths.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var config = require('./config.js'),
@@ -13,8 +11,6 @@ exports = module.exports = {
 
     ADDON_CONFIG_DIR: path.join(config.baseDir(), 'data/addons'),
 
-    DNS_IN_SYNC_FILE: path.join(config.baseDir(), 'data/dns_in_sync'),
-
     COLLECTD_APPCONFIG_DIR: path.join(config.baseDir(), 'data/collectd/collectd.conf.d'),
 
     DATA_DIR: path.join(config.baseDir(), 'data'),
@@ -26,6 +22,7 @@ exports = module.exports = {
 
     CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'data/box/avatar.png'),
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
+    FIRST_RUN_FILE: path.join(config.baseDir(), 'data/box/first_run'),
 
     UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
 

src/routes/apps.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -34,6 +32,11 @@ var apps = require('../apps.js'),
     util = require('util'),
     uuid = require('node-uuid');
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function removeInternalAppFields(app) {
     return {
         id: app.id,
@@ -49,7 +52,8 @@ function removeInternalAppFields(app) {
         portBindings: app.portBindings,
         iconUrl: app.iconUrl,
         fqdn: app.fqdn,
-        memoryLimit: app.memoryLimit
+        memoryLimit: app.memoryLimit,
+        altDomain: app.altDomain
     };
 }
 
@@ -125,13 +129,14 @@ function installApp(req, res, next) {
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
     // allow tests to provide an appId for testing
     var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
 
     debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j memoryLimit:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.memoryLimit, data.manifest);
 
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
+    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, data.altDomain || null, auditSource(req), function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -168,10 +173,11 @@ function configureApp(req, res, next) {
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+    if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
     debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j memoryLimit:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.memoryLimit);
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
+    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, data.altDomain || null, auditSource(req), function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -190,7 +196,7 @@ function restoreApp(req, res, next) {
 
     debug('Restore app id:%s', req.params.id);
 
-    apps.restore(req.params.id, function (error) {
+    apps.restore(req.params.id, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -224,7 +230,7 @@ function uninstallApp(req, res, next) {
 
     debug('Uninstalling app id:%s', req.params.id);
 
-    apps.uninstall(req.params.id, function (error) {
+    apps.uninstall(req.params.id, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error) return next(new HttpError(500, error));
 
@@ -274,7 +280,7 @@ function updateApp(req, res, next) {
 
     debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
 
-    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings, data.icon, function (error) {
+    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings || null, data.icon, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));

src/routes/backups.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -11,11 +9,14 @@ exports = module.exports = {
 var assert = require('assert'),
     backups = require('../backups.js'),
     BackupsError = require('../backups.js').BackupsError,
-    cloudron = require('../cloudron.js'),
-    CloudronError = require('../cloudron.js').CloudronError,
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function get(req, res, next) {
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
     if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
@@ -34,8 +35,8 @@ function get(req, res, next) {
 function create(req, res, next) {
     // note that cloudron.backup only waits for backup initiation and not for backup to complete
     // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    backups.backup(auditSource(req), function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
@@ -51,4 +52,3 @@ function download(req, res, next) {
         next(new HttpSuccess(200, result));
     });
 }
-

src/routes/clients.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {

src/routes/cloudron.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -15,15 +13,20 @@ exports = module.exports = {
 
 var assert = require('assert'),
     cloudron = require('../cloudron.js'),
-    config = require('../config.js'),
-    progress = require('../progress.js'),
-    mailer = require('../mailer.js'),
     CloudronError = cloudron.CloudronError,
+    config = require('../config.js'),
     debug = require('debug')('box:routes/cloudron'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    progress = require('../progress.js'),
+    mailer = require('../mailer.js'),
     superagent = require('superagent');
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 /**
  * Creating an admin user and activate the cloudron.
  *
@@ -50,7 +53,7 @@ function activate(req, res, next) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
     debug('activate: username:%s ip:%s', username, ip);
 
-    cloudron.activate(username, password, email, displayName, ip, function (error, info) {
+    cloudron.activate(username, password, email, displayName, ip, auditSource(req), function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
         if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
         if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
@@ -119,7 +122,7 @@ function getConfig(req, res, next) {
 
 function update(req, res, next) {
     // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    cloudron.updateToLatest(auditSource(req), function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
@@ -134,7 +137,8 @@ function feedback(req, res, next) {
     if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK &&
         req.body.type !== mailer.FEEDBACK_TYPE_TICKET &&
         req.body.type !== mailer.FEEDBACK_TYPE_APP_MISSING &&
-        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app_missing" or "app_error"'));
+        req.body.type !== mailer.FEEDBACK_TYPE_UPGRADE_REQUEST &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback", "app_missing", "app_error" or "upgrade_request"'));
     if (typeof req.body.subject !== 'string' || !req.body.subject) return next(new HttpError(400, 'subject must be string'));
     if (typeof req.body.description !== 'string' || !req.body.description) return next(new HttpError(400, 'description must be string'));
 

src/routes/developer.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -15,6 +13,11 @@ var developer = require('../developer.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
 function enabled(req, res, next) {
     developer.enabled(function (error, enabled) {
         if (enabled) return next();
@@ -25,8 +28,9 @@ function enabled(req, res, next) {
 function setEnabled(req, res, next) {
     if (typeof req.body.enabled !== 'boolean') return next(new HttpError(400, 'enabled must be boolean'));
 
-    developer.setEnabled(req.body.enabled, function (error) {
+    developer.setEnabled(req.body.enabled, auditSource(req), function (error) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, {}));
     });
 }
@@ -40,7 +44,7 @@ function login(req, res, next) {
         if (error) return next(new HttpError(500, error));
         if (!user) return next(new HttpError(401, 'Invalid credentials'));
 
-        developer.issueDeveloperToken(user, function (error, result) {
+        developer.issueDeveloperToken(user, auditSource(req), function (error, result) {
             if (error) return next(new HttpError(500, error));
 
             next(new HttpSuccess(200, { token: result.token, expiresAt: result.expiresAt }));
@@ -53,4 +57,4 @@ function apps(req, res, next) {
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { apps: result }));
     });
-}
+}

src/routes/eventlog.js

@@ -0,0 +1,23 @@
+'use strict';
+
+exports = module.exports = {
+    get: get
+};
+
+var eventlog = require('../eventlog.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess;
+
+function get(req, res, next) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    eventlog.getAllPaged(page, perPage, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { eventlogs: result }));
+    });
+}

src/routes/groups.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {

src/routes/index.js

@@ -6,10 +6,12 @@ exports = module.exports = {
     clients: require('./clients.js'),
     cloudron: require('./cloudron.js'),
     developer: require('./developer.js'),
+    eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
-    internal: require('./internal.js'),
     oauth2: require('./oauth2.js'),
+    profile: require('./profile.js'),
     settings: require('./settings.js'),
+    sysadmin: require('./sysadmin.js'),
     user: require('./user.js')
 };

src/routes/oauth2.js

@@ -1,34 +1,36 @@
-/* jslint  node:true */
-
 'use strict';
 
-var assert = require('assert'),
+var appdb = require('../appdb'),
     apps = require('../apps'),
+    assert = require('assert'),
     authcodedb = require('../authcodedb'),
     clientdb = require('../clientdb'),
     config = require('../config.js'),
     constants = require('../constants.js'),
     DatabaseError = require('../databaseerror'),
     debug = require('debug')('box:routes/oauth2'),
+    eventlog = require('../eventlog.js'),
+    hat = require('hat'),
     HttpError = require('connect-lastmile').HttpError,
     middleware = require('../middleware/index.js'),
     oauth2orize = require('oauth2orize'),
     passport = require('passport'),
     querystring = require('querystring'),
-    util = require('util'),
     session = require('connect-ensure-login'),
-    settings = require('../settings.js'),
     tokendb = require('../tokendb'),
-    appdb = require('../appdb'),
     url = require('url'),
     user = require('../user.js'),
     UserError = user.UserError,
-    hat = require('hat');
+    util = require('util');
+
+function auditSource(req, appId) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { authType: 'oauth', ip: ip, appId: appId };
+}
 
 // create OAuth 2.0 server
 var gServer = oauth2orize.createServer();
 
-
 // Register serialialization and deserialization functions.
 //
 // The client id is stored in the session and can thus be retrieved for each
@@ -58,7 +60,7 @@ gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client,
     var code = hat(256);
     var expiresAt = Date.now() + 60 * 60000; // 1 hour
 
-    authcodedb.add(code, client.id, user.username, expiresAt, function (error) {
+    authcodedb.add(code, client.id, user.id, expiresAt, function (error) {
         if (error) return callback(error);
 
         debug('grant code: new auth code for client %s code %s', client.id, code);
@@ -268,19 +270,56 @@ function passwordSentSite(req, res) {
     renderTemplate(res, 'password_reset_sent', { adminOrigin: config.adminOrigin(), title: 'Cloudron Password Reset' });
 }
 
-// -> GET /api/v1/session/password/setup.html
-function passwordSetupSite(req, res, next) {
-    if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
+function renderAccountSetupSite(res, req, userObject, error) {
+    renderTemplate(res, 'account_setup', {
+        adminOrigin: config.adminOrigin(),
+        user: userObject,
+        error: error,
+        csrf: req.csrfToken(),
+        resetToken: req.query.reset_token || req.body.resetToken,
+        title: 'Cloudron Password Setup'
+    });
+}
 
-    user.getByResetToken(req.query.reset_token, function (error, user) {
-        if (error) return next(new HttpError(401, 'Invalid reset_token'));
+// -> GET /api/v1/session/account/setup.html
+function accountSetupSite(req, res) {
+    if (!req.query.reset_token) return sendError(req, res, 'Missing Reset Token');
 
-        renderTemplate(res, 'password_setup', {
-            adminOrigin: config.adminOrigin(),
-            user: user,
-            csrf: req.csrfToken(),
-            resetToken: req.query.reset_token,
-            title: 'Cloudron Password Setup'
+    user.getByResetToken(req.query.reset_token, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Reset Token');
+
+        renderAccountSetupSite(res, req, userObject, '');
+    });
+}
+
+// -> POST /api/v1/session/account/setup
+function accountSetup(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'Missing resetToken'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'Missing password'));
+    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'Missing username'));
+    if (typeof req.body.displayName !== 'string') return next(new HttpError(400, 'Missing displayName'));
+
+    debug('acountSetup: with token %s.', req.body.resetToken);
+
+    user.getByResetToken(req.body.resetToken, function (error, userObject) {
+        if (error) return sendError(req, res, 'Invalid Reset Token');
+
+        userObject.username = req.body.username;
+        userObject.displayName = req.body.displayName;
+
+        user.update(userObject.id, userObject.username, userObject.email, userObject.displayName, auditSource(req), function (error) {
+            if (error && error.reason === UserError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
+            if (error) return next(new HttpError(500, error));
+
+            // setPassword clears the resetToken
+            user.setPassword(userObject.id, req.body.password, function (error, result) {
+                if (error && error.reason === UserError.BAD_PASSWORD) return renderAccountSetupSite(res, req, userObject, 'Password invalid');
+                if (error) return next(new HttpError(500, error));
+
+                res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
+            });
         });
     });
 }
@@ -314,6 +353,8 @@ function passwordReset(req, res, next) {
     user.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return next(new HttpError(401, 'Invalid resetToken'));
 
+        if (!userObject.username) return next(new HttpError(401, 'No username set'));
+
         // setPassword clears the resetToken
         user.setPassword(userObject.id, req.body.password, function (error, result) {
             if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
@@ -373,7 +414,10 @@ var authorization = [
         // Handle our different types of oauth clients
         var type = req.oauth2.client.type;
 
-        if (type === clientdb.TYPE_ADMIN) return next();
+        if (type === clientdb.TYPE_ADMIN) {
+            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, 'admin'), { userId: req.oauth2.user.id });
+            return next();
+        }
         if (type === clientdb.TYPE_EXTERNAL) return next();
         if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');
 
@@ -384,6 +428,8 @@ var authorization = [
                 if (error) return sendError(req, res, 'Internal error');
                 if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
 
+                eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, appObject.id), { userId: req.oauth2.user.id });
+
                 next();
             });
         });
@@ -460,8 +506,9 @@ exports = module.exports = {
     passwordResetRequest: passwordResetRequest,
     passwordSentSite: passwordSentSite,
     passwordResetSite: passwordResetSite,
-    passwordSetupSite: passwordSetupSite,
     passwordReset: passwordReset,
+    accountSetupSite: accountSetupSite,
+    accountSetup: accountSetup,
     authorization: authorization,
     token: token,
     scope: scope,

src/routes/profile.js

@@ -0,0 +1,82 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    update: update,
+    changePassword: changePassword
+};
+
+var assert = require('assert'),
+    groups = require('../groups.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    user = require('../user.js'),
+    tokendb = require('../tokendb.js'),
+    UserError = user.UserError;
+
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    var result = {};
+    result.id = req.user.id;
+    result.tokenType = req.user.tokenType;
+
+    if (req.user.tokenType === tokendb.TYPE_USER || req.user.tokenType === tokendb.TYPE_DEV) {
+        result.username = req.user.username;
+        result.email = req.user.email;
+        result.displayName = req.user.displayName;
+
+        groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
+            if (error) return next(new HttpError(500, error));
+
+            result.admin = isAdmin;
+
+            next(new HttpSuccess(200, result));
+        });
+    } else {
+        next(new HttpSuccess(200, result));
+    }
+}
+
+function update(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+    assert.strictEqual(typeof req.body, 'object');
+
+    if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
+
+    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+
+    user.update(req.user.id, req.user.username, req.body.email || req.user.email, req.body.displayName || req.user.displayName, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function changePassword(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires the users old password.'));
+    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'API call requires the users new password.'));
+
+    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+
+    user.setPassword(req.user.id, req.body.newPassword, function (error) {
+        if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/settings.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {

src/routes/sysadmin.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
@@ -8,9 +6,11 @@ exports = module.exports = {
     retire: retire
 };
 
-var cloudron = require('../cloudron.js'),
+var backups = require('../backups.js'),
+    BackupsError = require('../backups.js').BackupsError,
+    cloudron = require('../cloudron.js'),
     CloudronError = require('../cloudron.js').CloudronError,
-    debug = require('debug')('box:routes/internal'),
+    debug = require('debug')('box:routes/sysadmin'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
@@ -19,8 +19,9 @@ function backup(req, res, next) {
 
     // note that cloudron.backup only waits for backup initiation and not for backup to complete
     // backup progress can be checked up ny polling the progress api call
-    cloudron.backup(function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+    var auditSource = { userId: null, username: 'sysadmin' };
+    backups.backup(auditSource, function (error) {
+        if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, {}));
@@ -31,7 +32,8 @@ function update(req, res, next) {
     debug('triggering update');
 
     // this only initiates the update, progress can be checked via the progress route
-    cloudron.updateToLatest(function (error) {
+    var auditSource = { userId: null, username: 'sysadmin' };
+    cloudron.updateToLatest(auditSource, function (error) {
         if (error && error.reason === CloudronError.ALREADY_UPTODATE) return next(new HttpError(422, error.message));
         if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));

src/routes/test/apps-test.js

@@ -58,8 +58,8 @@ var APP_MANIFEST_1 = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-a
 APP_MANIFEST_1.dockerImage = TEST_IMAGE;
 APP_MANIFEST_1.singleUser = true;
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
-var USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
+var USER_1_ID = null, USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;
 
@@ -182,6 +182,8 @@ describe('Apps', function () {
                        .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
 
+                    USER_1_ID = res.body.id;
+
                     callback(null);
                 });
             },
@@ -190,7 +192,7 @@ describe('Apps', function () {
                 token_1 = tokendb.generateToken();
 
                 // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, tokendb.PREFIX_USER + USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
             },
 
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
@@ -828,6 +830,34 @@ describe('Apps', function () {
             });
         });
 
+        it('installation - mongodb addon config', function (done) {
+            var appContainer = docker.getContainer(appEntry.containerId);
+            appContainer.inspect(function (error, data) {
+                var mongodbUrl = null;
+                data.Config.Env.forEach(function (env) { if (env.indexOf('MONGODB_URL=') === 0) mongodbUrl = env.split('=')[1]; });
+                expect(mongodbUrl).to.be.ok();
+
+                var urlp = url.parse(mongodbUrl);
+                var username = urlp.auth.split(':')[0];
+                var password = urlp.auth.split(':')[1];
+                var dbname = urlp.path.substr(1);
+
+                expect(data.Config.Env).to.contain('MONGODB_PORT=27017');
+                expect(data.Config.Env).to.contain('MONGODB_HOST=mongodb');
+                expect(data.Config.Env).to.contain('MONGODB_USERNAME=' + username);
+                expect(data.Config.Env).to.contain('MONGODB_PASSWORD=' + password);
+                expect(data.Config.Env).to.contain('MONGODB_DATABASE=' + dbname);
+
+                var cmd = util.format('mongo --quiet -u %s -p %s %s:%s/%s --eval "db.collection.insert({ item: 34 })"',
+                    username, password, 'mongodb', 27017, dbname);
+
+                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error) {
+                    expect(!error).to.be.ok();
+                    done();
+                });
+            });
+        });
+
         it('installation - scheduler', function (done) {
             async.retry({ times: 100, interval: 1000 }, function (retryCallback) {
                 if (fs.existsSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env')) return retryCallback();

src/routes/test/backups-test.js

@@ -19,7 +19,7 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;
 
 var server;
@@ -53,7 +53,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
         },
 
         function createSettings(callback) {
@@ -95,7 +95,7 @@ describe('Backups API', function () {
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
                         .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
-                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey' } });
 
             superagent.post(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })

src/routes/test/clients-test.js

@@ -8,6 +8,7 @@
 
 var async = require('async'),
     config = require('../../config.js'),
+    clientdb = require('../../clientdb.js'),
     database = require('../../database.js'),
     oauth2 = require('../oauth2.js'),
     expect = require('expect.js'),
@@ -20,7 +21,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 function cleanup(done) {
@@ -173,6 +174,8 @@ describe('OAuth Clients API', function () {
                     expect(result.body.redirectURI).to.be.a('string');
                     expect(result.body.clientSecret).to.be.a('string');
                     expect(result.body.scope).to.be.a('string');
+                    expect(result.body.type).to.equal(clientdb.TYPE_EXTERNAL);
+
                     done();
                 });
             });
@@ -412,7 +415,7 @@ describe('Clients', function () {
             server.start.bind(server),
             database._clear.bind(null),
             function (callback) {
-            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
                 var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
 
                 superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
@@ -427,7 +430,16 @@ describe('Clients', function () {
                     // stash for further use
                     token = result.body.token;
 
-                    callback();
+                    superagent.get(SERVER_URL + '/api/v1/profile')
+                            .query({ access_token: token })
+                            .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(result.statusCode).to.eql(200);
+
+                        USER_0.id = result.body.id;
+
+                        callback();
+                    });
                 });
             }
         ], done);
@@ -531,7 +543,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.username);
+                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
 
                 done();
             });
@@ -584,7 +596,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.username);
+                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
 
                 superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin/tokens')
                 .query({ access_token: token })

src/routes/test/cloudron-test.js

@@ -18,7 +18,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 var server;
@@ -233,7 +233,9 @@ describe('Cloudron', function () {
         });
 
         it('succeeds', function (done) {
-            var scope = nock(config.apiServerOrigin()).get('/api/v1/boxes/localhost?token=' + config.token()).reply(200, { box: { region: 'sfo', size: '1gb' }});
+            var scope = nock(config.apiServerOrigin())
+                  .get('/api/v1/boxes/localhost?token=' + config.token())
+                  .reply(200, { box: { region: 'sfo', size: '1gb' }, user: { }});
 
             superagent.get(SERVER_URL + '/api/v1/cloudron/config')
                    .query({ access_token: token })

src/routes/test/developer-test.js

@@ -17,7 +17,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 var server;
@@ -297,7 +297,16 @@ describe('Developer API', function () {
 
         it('fails with unknown username', function (done) {
             superagent.post(SERVER_URL + '/api/v1/developer/login')
-                   .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                   .send({ username: USERNAME + USERNAME, password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails with unknown email', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: USERNAME + EMAIL, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(401);
                 done();
@@ -324,6 +333,17 @@ describe('Developer API', function () {
             });
         });
 
+        it('with uppercase username succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.expiresAt).to.be.a('number');
+                expect(result.body.token).to.be.a('string');
+                done();
+            });
+        });
+
         it('with email succeeds', function (done) {
             superagent.post(SERVER_URL + '/api/v1/developer/login')
                    .send({ username: EMAIL, password: PASSWORD })
@@ -334,5 +354,16 @@ describe('Developer API', function () {
                 done();
             });
         });
+
+        it('with uppercase email succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/developer/login')
+                   .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.expiresAt).to.be.a('number');
+                expect(result.body.token).to.be.a('string');
+                done();
+            });
+        });
     });
 });

src/routes/test/groups-test.js

@@ -21,9 +21,10 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1337', EMAIL_1 ='happy@me.com';
 var token, token_1 = null;
+var userId, userId_1 = null;
 
 var server;
 function setup(done) {
@@ -48,7 +49,16 @@ function setup(done) {
                 // stash token for further use
                 token = result.body.token;
 
-                callback();
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                      .query({ access_token: token })
+                      .end(function (error, result) {
+                    expect(result).to.be.ok();
+                    expect(result.statusCode).to.eql(200);
+
+                    userId = result.body.id;
+
+                    callback();
+                });
             });
         },
         function (callback) {
@@ -60,9 +70,10 @@ function setup(done) {
                 expect(result.statusCode).to.eql(201);
 
                 token_1 = tokendb.generateToken();
+                userId_1 = result.body.id;
 
                 // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, userId_1, 'test-client-id',  Date.now() + 100000, '*', callback);
             });
         }
   ], done);
@@ -168,7 +179,7 @@ describe('Groups API', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.name).to.be('admin');
                 expect(result.body.userIds.length).to.be(1);
-                expect(result.body.userIds[0]).to.be(USERNAME);
+                expect(result.body.userIds[0]).to.be(userId);
                 done();
             });
         });
@@ -213,7 +224,7 @@ describe('Groups API', function () {
         });
 
         it('cannot add user to invalid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'something' ]})
                   .end(function (error, result) {
@@ -223,7 +234,7 @@ describe('Groups API', function () {
         });
 
         it('can add user to valid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -233,7 +244,7 @@ describe('Groups API', function () {
         });
 
         it('can remove last user from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'group0', 'group1' ]})
                   .end(function (error, result) {

src/routes/test/oauth2-test.js

@@ -138,9 +138,9 @@ describe('OAuth2', function () {
     describe('flow', function () {
         var USER_0 = {
             id: uuid.v4(),
-            username: 'someusername',
+            username: 'someUSERname',
             password: '@#45Strongpassword',
-            email: 'some@email.com',
+            email: 'some@EMAIL.com',
             salt: 'somesalt',
             createdAt: (new Date()).toUTCString(),
             modifiedAt: (new Date()).toUTCString(),
@@ -155,7 +155,8 @@ describe('OAuth2', function () {
             location: 'test',
             portBindings: {},
             accessRestriction: null,
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         var APP_1 = {
@@ -165,7 +166,8 @@ describe('OAuth2', function () {
             location: 'test1',
             portBindings: {},
             accessRestriction: { users: [ 'foobar' ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         var APP_2 = {
@@ -175,7 +177,8 @@ describe('OAuth2', function () {
             location: 'test2',
             portBindings: {},
             accessRestriction: { users: [ USER_0.id ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         var APP_3 = {
@@ -185,7 +188,8 @@ describe('OAuth2', function () {
             location: 'test3',
             portBindings: {},
             accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         // unknown app
@@ -308,12 +312,12 @@ describe('OAuth2', function () {
                 clientdb.add.bind(null, CLIENT_7.id, CLIENT_7.appId, CLIENT_7.type, CLIENT_7.clientSecret, CLIENT_7.redirectURI, CLIENT_7.scope),
                 clientdb.add.bind(null, CLIENT_8.id, CLIENT_8.appId, CLIENT_8.type, CLIENT_8.clientSecret, CLIENT_8.redirectURI, CLIENT_8.scope),
                 clientdb.add.bind(null, CLIENT_9.id, CLIENT_9.appId, CLIENT_9.type, CLIENT_9.clientSecret, CLIENT_9.redirectURI, CLIENT_9.scope),
-                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
-                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
-                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit),
-                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit),
+                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, APP_0.altDomain),
+                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, APP_1.altDomain),
+                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, APP_2.altDomain),
+                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit, APP_3.altDomain),
                 function (callback) {
-                    user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, function (error, userObject) {
+                    user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, null /* source */, function (error, userObject) {
                         expect(error).to.not.be.ok();
 
                         // update the global objects to reflect the new user id
@@ -883,7 +887,14 @@ describe('OAuth2', function () {
                         expect(foo.access_token).to.be.a('string');
                         expect(foo.token_type).to.eql('Bearer');
 
-                        done();
+                        // Ensure the token is also usable
+                        superagent.get(SERVER_URL + '/api/v1/profile?access_token=' + foo.access_token, function (error, result) {
+                            expect(error).to.not.be.ok();
+                            expect(result.status).to.eql(200);
+                            expect(result.body.username).to.equal(USER_0.username.toLowerCase());
+
+                            done();
+                        });
                     });
                 });
             });
@@ -1263,7 +1274,14 @@ describe('OAuth2', function () {
                             expect(body.access_token).to.be.a('string');
                             expect(body.token_type).to.eql('Bearer');
 
-                            done();
+                            // Ensure the token is also usable
+                            superagent.get(SERVER_URL + '/api/v1/profile?access_token=' + body.access_token, function (error, result) {
+                                expect(error).to.not.be.ok();
+                                expect(result.status).to.eql(200);
+                                expect(result.body.username).to.equal(USER_0.username.toLowerCase());
+
+                                done();
+                            });
                         });
                     });
                 });
@@ -1318,31 +1336,33 @@ describe('Password', function () {
         it('reset request succeeds', function (done) {
             superagent.get(SERVER_URL + '/api/v1/session/password/resetRequest.html')
             .end(function (error, result) {
-                expect(result.text.indexOf('<!-- tester -->')).to.not.equal(-1);
                 expect(result.statusCode).to.equal(200);
+                expect(result.text.indexOf('<!-- tester -->')).to.not.equal(-1);
                 done();
             });
         });
 
         it('setup fails due to missing reset_token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/session/password/setup.html')
+            superagent.get(SERVER_URL + '/api/v1/session/account/setup.html')
             .end(function (error, result) {
-                expect(result.statusCode).to.equal(400);
+                expect(result.statusCode).to.equal(200);
+                expect(result.text.indexOf('<!-- error tester -->')).to.not.equal(-1);
                 done();
             });
         });
 
         it('setup fails due to invalid reset_token', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/session/password/setup.html')
+            superagent.get(SERVER_URL + '/api/v1/session/account/setup.html')
             .query({ reset_token: hat(256) })
             .end(function (error, result) {
-                expect(result.statusCode).to.equal(401);
+                expect(result.statusCode).to.equal(200);
+                expect(result.text.indexOf('<!-- error tester -->')).to.not.equal(-1);
                 done();
             });
         });
 
         it('setup succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/session/password/setup.html')
+            superagent.get(SERVER_URL + '/api/v1/session/account/setup.html')
             .query({ reset_token: USER_0.resetToken })
             .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);

src/routes/test/profile-test.js

@@ -0,0 +1,293 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var config = require('../../config.js'),
+    database = require('../../database.js'),
+    tokendb = require('../../tokendb.js'),
+    expect = require('expect.js'),
+    groups = require('../../groups.js'),
+    mailer = require('../../mailer.js'),
+    superagent = require('superagent'),
+    nock = require('nock'),
+    server = require('../../server.js'),
+    userdb = require('../../userdb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME_0 = 'superaDmIn', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silLY@me.com', EMAIL_0_NEW = 'stupID@me.com', DISPLAY_NAME_0_NEW = 'New Name';
+var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'taO@zen.mac';
+var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'USER@foo.bar', EMAIL_2_NEW = 'happy@ME.com';
+var USERNAME_3 = 'userTheThird', EMAIL_3 = 'user3@FOO.bar';
+
+describe('Profile API', function () {
+    this.timeout(5000);
+
+    var user_0, user_1, user_2, user_3 = null;
+    var token_0;
+    var token_1 = tokendb.generateToken();
+    var token_2 = tokendb.generateToken();
+    var token_3;
+
+    function setup(done) {
+        server.start(function (error) {
+            expect(!error).to.be.ok();
+
+            mailer._clearMailQueue();
+
+            database._clear(function (error) {
+                expect(error).to.eql(null);
+
+                var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                       .query({ setupToken: 'somesetuptoken' })
+                       .send({ username: USERNAME_0, password: PASSWORD, email: EMAIL_0 })
+                       .end(function (err, res) {
+                    expect(err).to.eql(null);
+                    expect(res.statusCode).to.equal(201);
+
+                    // stash for later use
+                    token_0 = res.body.token;
+
+                    expect(scope1.isDone()).to.be.ok();
+                    expect(scope2.isDone()).to.be.ok();
+
+                    done();
+                });
+            });
+        });
+    }
+
+    function cleanup(done) {
+        database._clear(function (error) {
+            expect(!error).to.be.ok();
+
+            mailer._clearMailQueue();
+
+            server.stop(done);
+        });
+    }
+
+    function checkMails(number, done) {
+        // mails are enqueued async
+        setTimeout(function () {
+            expect(mailer._getMailQueue().length).to.equal(number);
+            mailer._clearMailQueue();
+            done();
+        }, 500);
+    }
+
+    describe('get profile', function () {
+        before(setup);
+        after(cleanup);
+
+        it('fails without token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+
+                done();
+            });
+        });
+
+        it('fails with empty token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: '' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+
+                done();
+            });
+        });
+
+        it('fails with invalid token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: 'some token' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile/').query({ access_token: token_0 }).end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.username).to.equal(USERNAME_0.toLowerCase());
+                expect(result.body.email).to.equal(EMAIL_0.toLowerCase());
+                expect(result.body.admin).to.be.ok();
+
+                user_0 = result.body;
+
+                done();
+            });
+        });
+
+        it('fails with expired token', function (done) {
+            var token = tokendb.generateToken();
+            var expires = Date.now() - 2000; // 1 sec
+
+            tokendb.add(token, tokendb.PREFIX_USER + user_0.id, null, expires, '*', function (error) {
+                expect(error).to.not.be.ok();
+
+                superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {
+                    expect(result.statusCode).to.equal(401);
+
+                    done();
+                });
+            });
+        });
+
+        it('fails with invalid token in auth header', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile').set('Authorization', 'Bearer ' + 'x' + token_0).end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds with token in auth header', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/profile').set('Authorization', 'Bearer ' + token_0).end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.username).to.equal(USERNAME_0.toLowerCase());
+                expect(result.body.email).to.equal(EMAIL_0.toLowerCase());
+                expect(result.body.admin).to.be.ok();
+                expect(result.body.displayName).to.be.a('string');
+                expect(result.body.password).to.not.be.ok();
+                expect(result.body.salt).to.not.be.ok();
+                done();
+            });
+        });
+    });
+
+    describe('update', function () {
+        before(setup);
+        after(cleanup);
+
+        it('change email fails due to missing token', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile')
+                   .send({ email: EMAIL_0_NEW })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('change email fails due to invalid email', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({ email: 'foo@bar' })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('change user succeeds without email nor displayName', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({})
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('change email succeeds', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({ email: EMAIL_0_NEW })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                      .query({ access_token: token_0 })
+                      .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+                    expect(res.body.email).to.equal(EMAIL_0_NEW.toLowerCase());
+                    expect(res.body.admin).to.equal(true);
+                    expect(res.body.displayName).to.equal('');
+
+                    done();
+                });
+            });
+        });
+
+        it('change displayName succeeds', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile')
+                   .query({ access_token: token_0 })
+                   .send({ displayName: DISPLAY_NAME_0_NEW })
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                      .query({ access_token: token_0 })
+                      .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+                    expect(res.body.email).to.equal(EMAIL_0_NEW.toLowerCase());
+                    expect(res.body.admin).to.be.ok();
+                    expect(res.body.displayName).to.equal(DISPLAY_NAME_0_NEW);
+
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('password change', function () {
+        before(setup);
+        after(cleanup);
+
+        it('change password fails due to missing current password', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ newPassword: 'some wrong password' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('change password fails due to missing new password', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('change password fails due to wrong password', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: 'some wrong password', newPassword: 'MOre#$%34' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(403);
+                done();
+            });
+        });
+
+        it('change password fails due to invalid password', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: PASSWORD, newPassword: 'five' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('change password succeeds', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/profile/password')
+                   .query({ access_token: token_0 })
+                   .send({ password: PASSWORD, newPassword: 'MOre#$%34' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(204);
+                done();
+            });
+        });
+    });
+});

src/routes/test/settings-test.js

@@ -22,7 +22,7 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;
 
 var server;
@@ -56,7 +56,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
         }
     ], done);
 }

src/routes/test/simpleauth-test.js

@@ -21,7 +21,7 @@ describe('SimpleAuth API', function () {
     var SERVER_URL = 'http://localhost:' + config.get('port');
     var SIMPLE_AUTH_ORIGIN = 'http://localhost:' + config.get('simpleAuthPort');
 
-    var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+    var USERNAME = 'superaDMin', PASSWORD = 'Foobar?1337', EMAIL ='silly@ME.com';
 
     var APP_0 = {
         id: 'app0',
@@ -30,7 +30,8 @@ describe('SimpleAuth API', function () {
         location: 'test0',
         portBindings: {},
         accessRestriction: { users: [ 'foobar', 'someone'] },
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var APP_1 = {
@@ -39,8 +40,9 @@ describe('SimpleAuth API', function () {
         manifest: { version: '0.1.0', addons: { } },
         location: 'test1',
         portBindings: {},
-        accessRestriction: { users: [ 'foobar', USERNAME, 'someone' ] },
-        memoryLimit: 0
+        accessRestriction: { users: [ 'foobar', 'someone' ] },
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var APP_2 = {
@@ -50,7 +52,8 @@ describe('SimpleAuth API', function () {
         location: 'test2',
         portBindings: {},
         accessRestriction: null,
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var APP_3 = {
@@ -60,7 +63,8 @@ describe('SimpleAuth API', function () {
         location: 'test3',
         portBindings: {},
         accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
-        memoryLimit: 0
+        memoryLimit: 0,
+        altDomain: null
     };
 
     var CLIENT_0 = {
@@ -138,7 +142,14 @@ describe('SimpleAuth API', function () {
                     expect(scope1.isDone()).to.be.ok();
                     expect(scope2.isDone()).to.be.ok();
 
-                    callback();
+                    superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: result.body.token}).end(function (error, result) {
+                        expect(error).to.not.be.ok();
+                        expect(result.statusCode).to.eql(200);
+
+                        APP_1.accessRestriction.users.push(result.body.id);
+
+                        callback();
+                    });
                 });
             },
 
@@ -148,10 +159,10 @@ describe('SimpleAuth API', function () {
             clientdb.add.bind(null, CLIENT_3.id, CLIENT_3.appId, CLIENT_3.type, CLIENT_3.clientSecret, CLIENT_3.redirectURI, CLIENT_3.scope),
             clientdb.add.bind(null, CLIENT_4.id, CLIENT_4.appId, CLIENT_4.type, CLIENT_4.clientSecret, CLIENT_4.redirectURI, CLIENT_4.scope),
             clientdb.add.bind(null, CLIENT_5.id, CLIENT_5.appId, CLIENT_5.type, CLIENT_5.clientSecret, CLIENT_5.redirectURI, CLIENT_5.scope),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit),
-            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, APP_0.altDomain),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, APP_1.altDomain),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, APP_2.altDomain),
+            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit, APP_3.altDomain)
         ], done);
     });
 
@@ -317,7 +328,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });
@@ -349,7 +361,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });
@@ -381,7 +394,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });
@@ -413,7 +427,8 @@ describe('SimpleAuth API', function () {
                 .end(function (error, result) {
                     expect(error).to.be(null);
                     expect(result.body).to.be.an('object');
-                    expect(result.body.username).to.eql(USERNAME);
+                    expect(result.body.username).to.eql(USERNAME.toLowerCase());
+                    expect(result.body.email).to.eql(EMAIL.toLowerCase());
 
                     done();
                 });

src/routes/test/start_addons.sh

@@ -68,7 +68,7 @@ start_mongodb() {
 start_mail() {
     docker rm -f mail 2>/dev/null 1>&2 || true
 
-    docker run -dP --name=mail -e DOMAIN_NAME="localhost" \
+    docker run -dP --name=mail -e MAIL_SERVER_NAME="server.local" -e MAIL_DOMAIN="server.local" \
         --read-only -v /tmp -v /run \
         -v /tmp/maildata:/app/data "${MAIL_IMAGE}" >/dev/null
 }

src/routes/test/sysadmin-test.js

@@ -0,0 +1,97 @@
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var appdb = require('../../appdb.js'),
+    async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    settings = require('../../settings.js'),
+    nock = require('nock');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.setVersion('1.2.3');
+
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        },
+
+        function addApp(callback) {
+            var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+        },
+
+        function createSettings(callback) {
+            settings.setBackupConfig({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' }, callback);
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Internal API', function () {
+    before(setup);
+    after(cleanup);
+
+    describe('backup', function () {
+        it('succeeds', function (done) {
+            var scope = nock(config.apiServerOrigin())
+                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey' } });
+
+            superagent.post(config.sysadminOrigin() + '/api/v1/backup')
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(202);
+
+                function checkAppstoreServerCalled() {
+                    if (scope.isDone()) {
+                        return done();
+                    }
+
+                    setTimeout(checkAppstoreServerCalled, 100);
+                }
+
+                checkAppstoreServerCalled();
+            });
+        });
+    });
+});

src/routes/test/user-test.js

@@ -14,24 +14,22 @@ var config = require('../../config.js'),
     mailer = require('../../mailer.js'),
     superagent = require('superagent'),
     nock = require('nock'),
-    server = require('../../server.js'),
-    userdb = require('../../userdb.js');
+    server = require('../../server.js');
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME_0 = 'admin', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silly@me.com', EMAIL_0_NEW = 'stupid@me.com', DISPLAY_NAME_0_NEW = 'New Name';
-var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'tao@zen.mac';
-var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'user@foo.bar', EMAIL_2_NEW = 'happy@me.com';
-var USERNAME_3 = 'userTheThird', EMAIL_3 = 'user3@foo.bar';
+var USERNAME_0 = 'superaDmIn', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silLY@me.com', EMAIL_0_NEW = 'stupID@me.com', DISPLAY_NAME_0_NEW = 'New Name';
+var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'taO@zen.mac';
+var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'USER@foo.bar', EMAIL_2_NEW = 'happy@ME.com';
+var USERNAME_3 = 'userTheThird', EMAIL_3 = 'user3@FOO.bar';
 
-var server;
 function setup(done) {
     server.start(function (error) {
         expect(!error).to.be.ok();
 
         mailer._clearMailQueue();
 
-        userdb._clear(function (error) {
+        database._clear(function (error) {
             expect(error).to.eql(null);
 
             groups.create('somegroupid', done);
@@ -61,10 +59,9 @@ function checkMails(number, done) {
 describe('User API', function () {
     this.timeout(5000);
 
-    var user_0 = null;
+    var user_0, user_1, user_2, user_3 = null;
     var token = null;
     var token_1 = tokendb.generateToken();
-    var token_2 = tokendb.generateToken();
 
     before(setup);
     after(cleanup);
@@ -107,6 +104,7 @@ describe('User API', function () {
                .query({ setupToken: 'somesetuptoken' })
                .send({ username: USERNAME_0, password: PASSWORD, email: EMAIL_0 })
                .end(function (err, res) {
+            expect(err).to.eql(null);
             expect(res.statusCode).to.equal(201);
 
             // stash for later use
@@ -114,7 +112,16 @@ describe('User API', function () {
 
             expect(scope1.isDone()).to.be.ok();
             expect(scope2.isDone()).to.be.ok();
-            done(err);
+
+            superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {
+                expect(error).to.eql(null);
+                expect(result.status).to.equal(200);
+
+                // stash for further use
+                user_0 = result.body;
+
+                done();
+            });
         });
     });
 
@@ -127,17 +134,24 @@ describe('User API', function () {
         });
     });
 
-    it('can get userInfo with token', function (done) {
+    it('cannot get userInfo by username', function (done) {
         superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
         .query({ access_token: token })
         .end(function (err, res) {
-            expect(res.statusCode).to.equal(200);
-            expect(res.body.username).to.equal(USERNAME_0);
-            expect(res.body.email).to.equal(EMAIL_0);
-            expect(res.body.admin).to.be.ok();
+            expect(res.statusCode).to.equal(404);
 
-            // stash for further use
-            user_0 = res.body;
+            done();
+        });
+    });
+
+    it('can get userInfo with token', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
+        .query({ access_token: token })
+        .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+            expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
+            expect(res.body.admin).to.be.ok();
 
             done();
         });
@@ -162,19 +176,19 @@ describe('User API', function () {
     });
 
     it('can get userInfo with token', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(200);
-            expect(res.body.username).to.equal(USERNAME_0);
-            expect(res.body.email).to.equal(EMAIL_0);
+            expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+            expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
             expect(res.body.admin).to.be.ok();
             done();
         });
     });
 
     it('cannot get userInfo only with basic auth', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .auth(USERNAME_0, PASSWORD)
                .end(function (err, res) {
             expect(res.statusCode).to.equal(401);
@@ -183,7 +197,7 @@ describe('User API', function () {
     });
 
     it('cannot get userInfo with invalid token (token length)', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: 'x' + token })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(401);
@@ -192,7 +206,7 @@ describe('User API', function () {
     });
 
     it('cannot get userInfo with invalid token (wrong token)', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token.toUpperCase() })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(401);
@@ -201,12 +215,12 @@ describe('User API', function () {
     });
 
     it('can get userInfo with token in auth header', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .set('Authorization', 'Bearer ' + token)
                .end(function (err, res) {
             expect(res.statusCode).to.equal(200);
-            expect(res.body.username).to.equal(USERNAME_0);
-            expect(res.body.email).to.equal(EMAIL_0);
+            expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+            expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
             expect(res.body.admin).to.be.ok();
             expect(res.body.displayName).to.be.a('string');
             expect(res.body.password).to.not.be.ok();
@@ -216,7 +230,7 @@ describe('User API', function () {
     });
 
     it('cannot get userInfo with invalid token in auth header', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .set('Authorization', 'Bearer ' + 'x' + token)
                .end(function (err, res) {
             expect(res.statusCode).to.equal(401);
@@ -225,7 +239,7 @@ describe('User API', function () {
     });
 
     it('cannot get userInfo with invalid token (wrong token)', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                .set('Authorization', 'Bearer ' + 'x' + token.toUpperCase())
                .end(function (err, res) {
             expect(res.statusCode).to.equal(401);
@@ -239,13 +253,15 @@ describe('User API', function () {
         superagent.post(SERVER_URL + '/api/v1/users')
                .query({ access_token: token })
                .send({ username: USERNAME_1, email: EMAIL_1, invite: true })
-               .end(function (err, res) {
-            expect(err).to.not.be.ok();
-            expect(res.statusCode).to.equal(201);
+               .end(function (error, result) {
+            expect(error).to.not.be.ok();
+            expect(result.statusCode).to.equal(201);
+
+            user_1 = result.body;
 
             checkMails(2, function () {
               // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-              tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 10000, '*', done);
+              tokendb.add(token_1, tokendb.PREFIX_USER + user_1.id, 'test-client-id',  Date.now() + 10000, '*', done);
             });
         });
     });
@@ -266,7 +282,7 @@ describe('User API', function () {
     it('reinvite second user succeeds', function (done) {
         mailer._clearMailQueue();
 
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/invite')
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_1.id + '/invite')
                .query({ access_token: token })
                .send({})
                .end(function (err, res) {
@@ -277,13 +293,13 @@ describe('User API', function () {
     });
 
     it('set second user as admin succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/set_groups')
                .query({ access_token: token })
                .send({ groupIds: [ groups.ADMIN_GROUP_ID ] })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(204);
 
-            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+            superagent.get(SERVER_URL + '/api/v1/users/' + user_1.id)
                .query({ access_token: token })
                .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
@@ -295,7 +311,7 @@ describe('User API', function () {
     });
 
     it('remove itself from admins fails', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id + '/set_groups')
                .query({ access_token: token })
                .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
@@ -305,13 +321,13 @@ describe('User API', function () {
     });
 
     it('remove second user from admins succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/set_groups')
                .query({ access_token: token })
                .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(204);
 
-            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+            superagent.get(SERVER_URL + '/api/v1/users/' + user_1.id)
                .query({ access_token: token })
                .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
@@ -358,40 +374,32 @@ describe('User API', function () {
         superagent.post(SERVER_URL + '/api/v1/users')
                .query({ access_token: token })
                .send({ username: USERNAME_2, email: EMAIL_2, invite: false })
-               .end(function (error, res) {
-            expect(res.statusCode).to.equal(201);
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(201);
+
+            user_2 = result.body;
 
             superagent.post(SERVER_URL + '/api/v1/users')
                    .query({ access_token: token })
                    .send({ username: USERNAME_3, email: EMAIL_3, invite: true })
-                   .end(function (error, res) {
-                expect(res.statusCode).to.equal(201);
+                   .end(function (error, result) {
+                expect(result.statusCode).to.equal(201);
+
+                user_3 = result.body;
 
                 // one mail for first user creation, two mails for second user creation (see 'invite' flag)
-                checkMails(3, function () {
-                    // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                    tokendb.add(token_2, tokendb.PREFIX_USER + USERNAME_2, 'test-client-id',  Date.now() + 10000, '*', done);
-                });
+                checkMails(3, done);
             });
         });
     });
 
-    it('second user userInfo fails for first user', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
-               .query({ access_token: token_1 })
-               .end(function (error, result) {
-            expect(result.statusCode).to.equal(403);
-            done();
-        });
-    });
-
-    it('second user userInfo succeeds for second user', function (done) {
-        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
-               .query({ access_token: token_2 })
+    it('get userInfo succeeds for second user', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/users/' + user_2.id)
+               .query({ access_token: token })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(200);
-            expect(result.body.username).to.equal(USERNAME_2);
-            expect(result.body.email).to.equal(EMAIL_2);
+            expect(result.body.username).to.equal(USERNAME_2.toLowerCase());
+            expect(result.body.email).to.equal(EMAIL_2.toLowerCase());
             expect(result.body.admin).to.not.be.ok();
 
             done();
@@ -410,7 +418,7 @@ describe('User API', function () {
 
     it('list users fails for normal user', function (done) {
         superagent.get(SERVER_URL + '/api/v1/users')
-        .query({ access_token: token_2 })
+        .query({ access_token: token_1 })
         .end(function (error, res) {
             expect(res.statusCode).to.equal(403);
             done();
@@ -440,7 +448,7 @@ describe('User API', function () {
     });
 
     it('user removes himself is not allowed', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.del(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ password: PASSWORD })
                .end(function (err, res) {
@@ -450,7 +458,7 @@ describe('User API', function () {
     });
 
     it('admin cannot remove normal user without giving a password', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+        superagent.del(SERVER_URL + '/api/v1/users/' + user_1.id)
                .query({ access_token: token })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
@@ -459,7 +467,7 @@ describe('User API', function () {
     });
 
     it('admin cannot remove normal user with empty password', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+        superagent.del(SERVER_URL + '/api/v1/users/' + user_1.id)
                .query({ access_token: token })
                .send({ password: '' })
                .end(function (err, res) {
@@ -469,7 +477,7 @@ describe('User API', function () {
     });
 
     it('admin cannot remove normal user with giving wrong password', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+        superagent.del(SERVER_URL + '/api/v1/users/' + user_1.id)
                .query({ access_token: token })
                .send({ password: PASSWORD + PASSWORD })
                .end(function (err, res) {
@@ -479,7 +487,7 @@ describe('User API', function () {
     });
 
     it('admin removes normal user', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+        superagent.del(SERVER_URL + '/api/v1/users/' + user_1.id)
                .query({ access_token: token })
                .send({ password: PASSWORD })
                .end(function (err, res) {
@@ -489,7 +497,7 @@ describe('User API', function () {
     });
 
     it('admin removes himself should not be allowed', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.del(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ password: PASSWORD })
                .end(function (err, res) {
@@ -500,7 +508,7 @@ describe('User API', function () {
 
     // Change email
     it('change email fails due to missing token', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
                .send({ email: EMAIL_0_NEW })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(401);
@@ -509,7 +517,7 @@ describe('User API', function () {
     });
 
     it('change email fails due to invalid email', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ email: 'foo@bar' })
                .end(function (error, result) {
@@ -518,18 +526,8 @@ describe('User API', function () {
         });
     });
 
-    it('change email for other user fails', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
-               .query({ access_token: token_2 })
-               .send({ email: 'foobar@bar.baz' })
-               .end(function (error, result) {
-            expect(result.statusCode).to.equal(403);
-            done();
-        });
-    });
-
     it('change user succeeds without email nor displayName', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({})
                .end(function (error, result) {
@@ -538,19 +536,19 @@ describe('User API', function () {
         });
     });
 
-    it('change email for own user succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_2)
-               .query({ access_token: token_2 })
+    it('change email succeeds', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_2.id)
+               .query({ access_token: token })
                .send({ email: EMAIL_2_NEW })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(204);
 
-            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
-                  .query({ access_token: token_2 })
+            superagent.get(SERVER_URL + '/api/v1/users/' + user_2.id)
+                  .query({ access_token: token })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
-                expect(res.body.username).to.equal(USERNAME_2);
-                expect(res.body.email).to.equal(EMAIL_2_NEW);
+                expect(res.body.username).to.equal(USERNAME_2.toLowerCase());
+                expect(res.body.email).to.equal(EMAIL_2_NEW.toLowerCase());
                 expect(res.body.admin).to.equal(false);
                 expect(res.body.displayName).to.equal('');
 
@@ -560,18 +558,18 @@ describe('User API', function () {
     });
 
     it('change email as admin for other user succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_2.id)
                .query({ access_token: token })
                .send({ email: EMAIL_2 })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(204);
 
-            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+            superagent.get(SERVER_URL + '/api/v1/users/' + user_2.id)
                   .query({ access_token: token })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
-                expect(res.body.username).to.equal(USERNAME_2);
-                expect(res.body.email).to.equal(EMAIL_2);
+                expect(res.body.username).to.equal(USERNAME_2.toLowerCase());
+                expect(res.body.email).to.equal(EMAIL_2.toLowerCase());
                 expect(res.body.admin).to.equal(false);
                 expect(res.body.displayName).to.equal('');
 
@@ -581,18 +579,18 @@ describe('User API', function () {
     });
 
     it('change displayName succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ displayName: DISPLAY_NAME_0_NEW })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(204);
 
-            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+            superagent.get(SERVER_URL + '/api/v1/users/' + user_0.id)
                   .query({ access_token: token })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
-                expect(res.body.username).to.equal(USERNAME_0);
-                expect(res.body.email).to.equal(EMAIL_0);
+                expect(res.body.username).to.equal(USERNAME_0.toLowerCase());
+                expect(res.body.email).to.equal(EMAIL_0.toLowerCase());
                 expect(res.body.admin).to.be.ok();
                 expect(res.body.displayName).to.equal(DISPLAY_NAME_0_NEW);
 
@@ -600,55 +598,4 @@ describe('User API', function () {
             });
         });
     });
-
-    // Change password
-    it('change password fails due to missing current password', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
-               .query({ access_token: token })
-               .send({ newPassword: 'some wrong password' })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(400);
-            done();
-        });
-    });
-
-    it('change password fails due to missing new password', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
-               .query({ access_token: token })
-               .send({ password: PASSWORD })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(400);
-            done();
-        });
-    });
-
-    it('change password fails due to wrong password', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
-               .query({ access_token: token })
-               .send({ password: 'some wrong password', newPassword: 'MOre#$%34' })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(403);
-            done();
-        });
-    });
-
-    it('change password fails due to invalid password', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
-               .query({ access_token: token })
-               .send({ password: PASSWORD, newPassword: 'five' })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(400);
-            done();
-        });
-    });
-
-    it('change password succeeds', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
-               .query({ access_token: token })
-               .send({ password: PASSWORD, newPassword: 'MOre#$%34' })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(204);
-            done();
-        });
-    });
 });

src/routes/user.js

@@ -1,15 +1,11 @@
-/* jslint node:true */
-
 'use strict';
 
 exports = module.exports = {
-    profile: profile,
-    info: info,
+    get: get,
     update: update,
-    list: listUser,
-    create: createUser,
-    changePassword: changePassword,
-    remove: removeUser,
+    list: list,
+    create: create,
+    remove: remove,
     verifyPassword: verifyPassword,
     requireAdmin: requireAdmin,
     sendInvite: sendInvite,
@@ -25,45 +21,26 @@ var assert = require('assert'),
     tokendb = require('../tokendb.js'),
     UserError = user.UserError;
 
-function profile(req, res, next) {
-    assert.strictEqual(typeof req.user, 'object');
-
-    var result = {};
-    result.id = req.user.id;
-    result.tokenType = req.user.tokenType;
-
-    if (req.user.tokenType === tokendb.TYPE_USER || req.user.tokenType === tokendb.TYPE_DEV) {
-        result.username = req.user.username;
-        result.email = req.user.email;
-        result.displayName = req.user.displayName;
-
-        groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
-            if (error) return next(new HttpError(500, error));
-
-            result.admin = isAdmin;
-
-            next(new HttpSuccess(200, result));
-        });
-    } else {
-        next(new HttpSuccess(200, result));
-    }
+function auditSource(req) {
+    var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
+    return { ip: ip, username: req.user ? req.user.username : null, userId: req.user ? req.user.id : null };
 }
 
-function createUser(req, res, next) {
+function create(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be string'));
     if (typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
     if (typeof req.body.invite !== 'boolean') return next(new HttpError(400, 'invite must be boolean'));
+    if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be string'));
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
 
-    var username = req.body.username;
     var password = generatePassword();
     var email = req.body.email;
     var sendInvite = req.body.invite;
+    var username = req.body.username || '';
     var displayName = req.body.displayName || '';
 
-    user.create(username, password, email, displayName, { invitor: req.user, sendInvite: sendInvite }, function (error, user) {
+    user.create(username, password, email, displayName, auditSource(req), { invitor: req.user, sendInvite: sendInvite }, function (error, user) {
         if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, 'Invalid username'));
         if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, 'Invalid email'));
         if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, 'Invalid password'));
@@ -74,12 +51,13 @@ function createUser(req, res, next) {
         var userInfo = {
             id: user.id,
             username: user.username,
+            displayName: user.displayName,
             email: user.email,
             admin: user.admin,
             resetToken: user.resetToken
         };
 
-        next(new HttpSuccess(201, { userInfo: userInfo }));
+        next(new HttpSuccess(201, userInfo ));
     });
 }
 
@@ -98,8 +76,10 @@ function update(req, res, next) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        user.update(req.params.userId, result.username, req.body.email || result.email, req.body.displayName || result.displayName, function (error) {
+        user.update(req.params.userId, result.username, req.body.email || result.email, req.body.displayName || result.displayName, auditSource(req), function (error) {
+            if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, error.message));
             if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
+            if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
             if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
             if (error) return next(new HttpError(500, error));
 
@@ -108,33 +88,14 @@ function update(req, res, next) {
     });
 }
 
-function changePassword(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.user, 'object');
-
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires the users old password.'));
-    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'API call requires the users new password.'));
-
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
-
-    user.changePassword(req.user.username, req.body.password, req.body.newPassword, function (error) {
-        if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new HttpError(403, 'Wrong password'));
-        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(204));
-    });
-}
-
-function listUser(req, res, next) {
+function list(req, res, next) {
     user.list(function (error, result) {
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { users: result }));
     });
 }
 
-function info(req, res, next) {
+function get(req, res, next) {
     assert.strictEqual(typeof req.params.userId, 'string');
     assert.strictEqual(typeof req.user, 'object');
 
@@ -158,7 +119,7 @@ function info(req, res, next) {
     });
 }
 
-function removeUser(req, res, next) {
+function remove(req, res, next) {
     assert.strictEqual(typeof req.params.userId, 'string');
 
     // rules:
@@ -168,35 +129,33 @@ function removeUser(req, res, next) {
 
     if (req.user.id === req.params.userId) return next(new HttpError(403, 'Not allowed to remove yourself.'));
 
-    user.remove(req.params.userId, function (error) {
-        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+    user.get(req.params.userId, function (error, userObject) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(204));
+        user.remove(userObject, auditSource(req), function (error) {
+            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+            if (error) return next(new HttpError(500, error));
+
+            next(new HttpSuccess(204));
+        });
     });
 }
 
 function verifyPassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    // developers are allowed to through without password
+    // developers are allowed through without password
     if (req.user.tokenType === tokendb.TYPE_DEV) return next();
 
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires user password'));
 
-    groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
+    user.verifyWithUsername(req.user.username, req.body.password, function (error) {
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new HttpError(403, 'Password incorrect'));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Password incorrect'));
         if (error) return next(new HttpError(500, error));
 
-        // Only allow admins or users, operating on themselves
-        if (req.params.userId && !(req.user.id === req.params.userId || isAdmin)) return next(new HttpError(403, 'Not allowed'));
-
-        user.verify(req.user.username, req.body.password, function (error) {
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new HttpError(403, 'Password incorrect'));
-            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Password incorrect'));
-            if (error) return next(new HttpError(500, error));
-
-            next();
-        });
+        next();
     });
 }
 
@@ -214,11 +173,11 @@ function requireAdmin(req, res, next) {
 function sendInvite(req, res, next) {
     assert.strictEqual(typeof req.params.userId, 'string');
 
-    user.sendInvite(req.params.userId, function (error) {
+    user.sendInvite(req.params.userId, function (error, result) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200, {}));
+        next(new HttpSuccess(200, { resetToken: result }));
     });
 }
 

src/scripts/backupapp.sh

@@ -12,51 +12,56 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [ $# -lt 4 ]; then
-    echo "Usage: backupapp.sh <appid> <url> <url> <key> [aws session token]"
+if [ $# -lt 8 ]; then
+    echo "Usage: backupapp.sh <appid> <s3 config url> <s3 data url> <access key id> <access key> <session token> <region> <password>"
     exit 1
 fi
 
 readonly DATA_DIR="${HOME}/data"
 
-app_id="$1"
-backup_url="$2"
-backup_config_url="$3"
-backup_key="$4"
-session_token="$5" # unused since it seems to be part of the url query param in v4 signature
+# env vars used by the awscli
+readonly app_id="$1"
+readonly s3_config_url="$2"
+readonly s3_data_url="$3"
+export AWS_ACCESS_KEY_ID="$4"
+export AWS_SECRET_ACCESS_KEY="$5"
+export AWS_SESSION_TOKEN="$6"
+export AWS_DEFAULT_REGION="$7"
+readonly password="$8"
+
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
 
 btrfs subvolume snapshot -r "${app_data_dir}" "${app_data_snapshot}"
 
+# Upload config.json first because uploading tarball might take a lot of time, leading to token expiry
 for try in `seq 1 5`; do
-    echo "Uploading backup to ${backup_url} (try ${try})"
+    echo "Uploading config.json to ${s3_config_url} (try ${try})"
     error_log=$(mktemp)
 
-    headers=("-H" "Content-Type:")
-
-    if tar -cvzf - -C "${app_data_snapshot}" . \
-           | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
+    # use aws instead of curl because curl will always read entire stream memory to set Content-Length
+    # aws will do multipart upload
+    if cat "${app_data_snapshot}/config.json" \
+           |  aws s3 cp - "${s3_config_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"
 done
 
 if [[ ${try} -eq 5 ]]; then
-    echo "Backup failed uploading backup tarball"
+    echo "Backup failed uploading config.json"
+    btrfs subvolume delete "${app_data_snapshot}"
     exit 1
 fi
 
 for try in `seq 1 5`; do
-    echo "Uploading config.json to ${backup_config_url} (try ${try})"
+    echo "Uploading backup to ${s3_data_url} (try ${try})"
     error_log=$(mktemp)
 
-    headers=("-H" "Content-Type:")
-
-    if cat "${app_data_snapshot}/config.json" \
-           | curl --fail -X PUT ${headers[@]} --data @- "${backup_config_url}" 2>"${error_log}"; then
+    if tar -czf - -C "${app_data_snapshot}" . \
+           | openssl aes-256-cbc -e -pass "pass:${password}" \
+           |  aws s3 cp - "${s3_data_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"
@@ -65,7 +70,7 @@ done
 btrfs subvolume delete "${app_data_snapshot}"
 
 if [[ ${try} -eq 5 ]]; then
-    echo "Backup failed uploading config.json"
+    echo "Backup failed uploading backup tarball"
     exit 1
 else
     echo "Backup successful"

src/scripts/backupbox.sh

@@ -12,14 +12,18 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [ $# -lt 2 ]; then
-    echo "Usage: backupbox.sh <url> <key> [aws session token]"
+if [ $# -lt 6 ]; then
+    echo "Usage: backupbox.sh <s3 url> <access key id> <access key> <session token> <region> <password>"
     exit 1
 fi
 
-backup_url="$1"
-backup_key="$2"
-session_token="$3"
+# env vars used by the awscli
+s3_url="$1"
+export AWS_ACCESS_KEY_ID="$2"
+export AWS_SECRET_ACCESS_KEY="$3"
+export AWS_SESSION_TOKEN="$4"
+export AWS_DEFAULT_REGION="$5"
+password="$6"
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"
@@ -31,19 +35,14 @@ echo "Snapshoting backup as backup-${now}"
 btrfs subvolume snapshot -r "${BOX_DATA_DIR}" "${box_snapshot_dir}"
 
 for try in `seq 1 5`; do
-    echo "Uploading backup to ${backup_url} (try ${try})"
+    echo "Uploading backup to ${s3_url} (try ${try})"
     error_log=$(mktemp)
 
-    headers=("-H" "Content-Type:")
-
-    # federated tokens in CaaS case need session token
-    if [ ! -z "$session_token" ]; then
-        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
-    fi
-
-    if tar -cvzf - -C "${box_snapshot_dir}" . \
-           | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
+    # use aws instead of curl because curl will always read entire stream memory to set Content-Length
+    # aws will do multipart upload
+    if tar -czf - -C "${box_snapshot_dir}" . \
+           | openssl aes-256-cbc -e -pass "pass:${password}" \
+           | aws s3 cp - "${s3_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/backupswap.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# -eq 0 ]]; then
-    echo "No arguments supplied"
-    exit 1
-fi
-
-if [[ "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-BACKUP_SWAP_FILE="/backup.swap"
-
-if [[ "$1" == "--on" ]]; then
-    echo "Mounting backup swap"
-
-    if ! swapon -s | grep -q "${BACKUP_SWAP_FILE}"; then
-        swapon "${BACKUP_SWAP_FILE}"
-    else
-        echo "Backup swap already mounted"
-    fi
-fi
-
-if [[ "$1" == "--off" ]]; then
-    echo "Unmounting backup swap"
-
-    if swapon -s | grep -q "${BACKUP_SWAP_FILE}"; then
-        swapoff "${BACKUP_SWAP_FILE}"
-    else
-        echo "Backup swap was not mounted"
-    fi
-fi
-

src/scripts/collectlogs.sh

@@ -21,7 +21,7 @@ readonly program_name=$1
 
 echo "${program_name}.log"
 echo "-------------------"
-journalctl --all --no-pager -u ${program_name} -n 100
+journalctl --all --no-pager -u ${program_name} -n 300
 echo
 echo
 echo "dmesg"

src/server.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -11,10 +9,12 @@ var assert = require('assert'),
     async = require('async'),
     auth = require('./auth.js'),
     certificates = require('./certificates.js'),
+    clients = require('./clients.js'),
     cloudron = require('./cloudron.js'),
     cron = require('./cron.js'),
     config = require('./config.js'),
     database = require('./database.js'),
+    eventlog = require('./eventlog.js'),
     express = require('express'),
     http = require('http'),
     mailer = require('./mailer.js'),
@@ -25,7 +25,7 @@ var assert = require('assert'),
     taskmanager = require('./taskmanager.js');
 
 var gHttpServer = null;
-var gInternalHttpServer = null;
+var gSysadminHttpServer = null;
 
 function initializeExpressSync() {
     var app = express();
@@ -67,12 +67,12 @@ function initializeExpressSync() {
     var multipart = middleware.multipart({ maxFieldsSize: FIELD_LIMIT, limit: FILE_SIZE_LIMIT, timeout: FILE_TIMEOUT });
 
     // scope middleware implicitly also adds bearer token verification
-    var rootScope = routes.oauth2.scope('root');
-    var profileScope = routes.oauth2.scope('profile');
-    var usersScope = routes.oauth2.scope('users');
-    var appsScope = routes.oauth2.scope('apps');
-    var developerScope = routes.oauth2.scope('developer');
-    var settingsScope = routes.oauth2.scope('settings');
+    var rootScope = routes.oauth2.scope(clients.SCOPE_ROOT);
+    var profileScope = routes.oauth2.scope(clients.SCOPE_PROFILE);
+    var usersScope = routes.oauth2.scope(clients.SCOPE_USERS);
+    var appsScope = routes.oauth2.scope(clients.SCOPE_APPS);
+    var developerScope = routes.oauth2.scope(clients.SCOPE_DEVELOPER);
+    var settingsScope = routes.oauth2.scope(clients.SCOPE_SETTINGS);
 
     // csrf protection
     var csrf = routes.oauth2.csrf;
@@ -98,20 +98,20 @@ function initializeExpressSync() {
     // feedback
     router.post('/api/v1/cloudron/feedback', usersScope, routes.cloudron.feedback);
 
-    router.get ('/api/v1/profile', profileScope, routes.user.profile);
+    // profile api, working off the user behind the provided token
+    router.get ('/api/v1/profile', profileScope, routes.profile.get);
+    router.put ('/api/v1/profile', profileScope, routes.profile.update);
+    router.put ('/api/v1/profile/password', profileScope, routes.user.verifyPassword, routes.profile.changePassword);
 
     // user routes only for admins
     router.get ('/api/v1/users', usersScope, routes.user.requireAdmin, routes.user.list);
     router.post('/api/v1/users', usersScope, routes.user.requireAdmin, routes.user.create);
+    router.get ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.get);
     router.del ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.user.remove);
+    router.put ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.update);
     router.put ('/api/v1/users/:userId/set_groups', usersScope, routes.user.requireAdmin, routes.user.setGroups);
     router.post('/api/v1/users/:userId/invite', usersScope, routes.user.requireAdmin, routes.user.sendInvite);
 
-    // user routes for admins and users operating on their own account
-    router.get ('/api/v1/users/:userId', usersScope, routes.user.info);
-    router.put ('/api/v1/users/:userId', usersScope, routes.user.update);
-    router.post('/api/v1/users/:userId/password', usersScope, routes.user.changePassword); // changePassword verifies password
-
     // Group management
     router.get ('/api/v1/groups', usersScope, routes.user.requireAdmin, routes.groups.list);
     router.post('/api/v1/groups', usersScope, routes.user.requireAdmin, routes.groups.create);
@@ -126,9 +126,10 @@ function initializeExpressSync() {
     router.get ('/api/v1/session/password/resetRequest.html', csrf, routes.oauth2.passwordResetRequestSite);
     router.post('/api/v1/session/password/resetRequest', csrf, routes.oauth2.passwordResetRequest);
     router.get ('/api/v1/session/password/sent.html', routes.oauth2.passwordSentSite);
-    router.get ('/api/v1/session/password/setup.html', csrf, routes.oauth2.passwordSetupSite);
     router.get ('/api/v1/session/password/reset.html', csrf, routes.oauth2.passwordResetSite);
     router.post('/api/v1/session/password/reset', csrf, routes.oauth2.passwordReset);
+    router.get ('/api/v1/session/account/setup.html', csrf, routes.oauth2.accountSetupSite);
+    router.post('/api/v1/session/account/setup', csrf, routes.oauth2.accountSetup);
 
     // oauth2 routes
     router.get ('/api/v1/oauth/dialog/authorize', routes.oauth2.authorization);
@@ -176,6 +177,9 @@ function initializeExpressSync() {
     router.post('/api/v1/settings/certificate',        settingsScope, routes.settings.setCertificate);
     router.post('/api/v1/settings/admin_certificate',  settingsScope, routes.settings.setAdminCertificate);
 
+    // eventlog route
+    router.get('/api/v1/eventlog', settingsScope, routes.eventlog.get);
+
     // backup routes
     router.get ('/api/v1/backups', settingsScope, routes.backups.get);
     router.post('/api/v1/backups', settingsScope, routes.backups.create);
@@ -211,7 +215,7 @@ function initializeExpressSync() {
 }
 
 // provides hooks for the 'installer'
-function initializeInternalExpressSync() {
+function initializeSysadminExpressSync() {
     var app = express();
     var httpServer = http.createServer(app);
 
@@ -221,7 +225,7 @@ function initializeInternalExpressSync() {
     var json = middleware.json({ strict: true, limit: QUERY_LIMIT }), // application/json
         urlencoded = middleware.urlencoded({ extended: false, limit: QUERY_LIMIT }); // application/x-www-form-urlencoded
 
-    if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('Box Internal :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
+    if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('Box Sysadmin :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
 
     var router = new express.Router();
     router.del = router.delete; // amend router.del for readability further on
@@ -233,10 +237,10 @@ function initializeInternalExpressSync() {
        .use(router)
        .use(middleware.lastMile());
 
-    // internal routes
-    router.post('/api/v1/backup', routes.internal.backup);
-    router.post('/api/v1/update', routes.internal.update);
-    router.post('/api/v1/retire', routes.internal.retire);
+    // Sysadmin routes
+    router.post('/api/v1/backup', routes.sysadmin.backup);
+    router.post('/api/v1/update', routes.sysadmin.update);
+    router.post('/api/v1/retire', routes.sysadmin.retire);
 
     return httpServer;
 }
@@ -246,7 +250,7 @@ function start(callback) {
     assert.strictEqual(gHttpServer, null, 'Server is already up and running.');
 
     gHttpServer = initializeExpressSync();
-    gInternalHttpServer = initializeInternalExpressSync();
+    gSysadminHttpServer = initializeSysadminExpressSync();
 
     async.series([
         auth.initialize,
@@ -257,7 +261,8 @@ function start(callback) {
         mailer.initialize,
         cron.initialize,
         gHttpServer.listen.bind(gHttpServer, config.get('port'), '127.0.0.1'),
-        gInternalHttpServer.listen.bind(gInternalHttpServer, config.get('internalPort'), '127.0.0.1')
+        gSysadminHttpServer.listen.bind(gSysadminHttpServer, config.get('sysadminPort'), '127.0.0.1'),
+        eventlog.add.bind(null, eventlog.ACTION_START, { userId: null, username: 'boot' }, { version: config.version() })
     ], callback);
 }
 
@@ -274,12 +279,12 @@ function stop(callback) {
         mailer.uninitialize,
         database.uninitialize,
         gHttpServer.close.bind(gHttpServer),
-        gInternalHttpServer.close.bind(gInternalHttpServer)
+        gSysadminHttpServer.close.bind(gSysadminHttpServer)
     ], function (error) {
         if (error) console.error(error);
 
         gHttpServer = null;
-        gInternalHttpServer = null;
+        gSysadminHttpServer = null;
 
         callback(null);
     });

src/simpleauth.js

@@ -14,6 +14,7 @@ var apps = require('./apps.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:src/simpleauth'),
+    eventlog = require('./eventlog.js'),
     express = require('express'),
     http = require('http'),
     HttpError = require('connect-lastmile').HttpError,
@@ -39,7 +40,7 @@ function loginLogic(clientId, username, password, callback) {
         // only allow simple auth clients
         if (clientObject.type !== clientdb.TYPE_SIMPLE_AUTH) return callback(new ClientsError(ClientsError.INVALID_CLIENT));
 
-        var authFunction = (username.indexOf('@') === -1) ? user.verify : user.verifyWithEmail;
+        var authFunction = (username.indexOf('@') === -1) ? user.verifyWithUsername : user.verifyWithEmail;
         authFunction(username, password, function (error, userObject) {
             if (error) return callback(error);
 
@@ -94,6 +95,8 @@ function login(req, res, next) {
         if (error && error.reason === AppsError.ACCESS_DENIED) return next(new HttpError(401, 'Forbidden'));
         if (error) return next(new HttpError(500, error));
 
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'simpleauth', clientId: req.body.clientId }, { userId: result.user.id });
+
         var tmp = {
             accessToken: result.accessToken,
             user: {

src/storage/caas.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    getSignedUploadUrl: getSignedUploadUrl,
-    getSignedDownloadUrl: getSignedDownloadUrl,
+    getRestoreUrl: getRestoreUrl,
 
     copyObject: copyObject,
 
+    getBackupCredentials: getBackupCredentials,
+
     getAllPaged: getAllPaged
 };
 
@@ -55,7 +56,7 @@ function getAllPaged(apiConfig, page, perPage, callback) {
     });
 }
 
-function getSignedUploadUrl(apiConfig, filename, callback) {
+function getRestoreUrl(apiConfig, filename, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -65,42 +66,18 @@ function getSignedUploadUrl(apiConfig, filename, callback) {
     getBackupCredentials(apiConfig, function (error, credentials) {
         if (error) return callback(error);
 
+        credentials.region = apiConfig.region; // use same region as where we uploaded
         var s3 = new AWS.S3(credentials);
 
         var params = {
             Bucket: apiConfig.bucket,
             Key: apiConfig.prefix + '/' + filename,
-            Expires: 60 * 30 /* 30 minutes */
-        };
-
-        var url = s3.getSignedUrl('putObject', params);
-
-        callback(null, { url: url, sessionToken: credentials.sessionToken });
-    });
-}
-
-function getSignedDownloadUrl(apiConfig, info, filename, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof info, 'object');
-    assert.strictEqual(typeof filename, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!info.bucket || !info.prefix) return new Error('Invalid configuration'); // prevent error in s3
-
-    getBackupCredentials(apiConfig, function (error, credentials) {
-        if (error) return callback(error);
-
-        var s3 = new AWS.S3(credentials);
-
-        var params = {
-            Bucket: info.bucket,
-            Key: info.prefix + '/' + filename,
-            Expires: 60 * 30 /* 30 minutes */
+            Expires: 60 * 60 /* 60 minutes */
         };
 
         var url = s3.getSignedUrl('getObject', params);
 
-        callback(null, { url: url, sessionToken: credentials.sessionToken });
+        callback(null, { url: url });
     });
 }
 

src/storage/s3.js

@@ -1,11 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    getSignedUploadUrl: getSignedUploadUrl,
-    getSignedDownloadUrl: getSignedDownloadUrl,
+    getRestoreUrl: getRestoreUrl,
 
     copyObject: copyObject,
-    getAllPaged: getAllPaged
+    getAllPaged: getAllPaged,
+
+    getBackupCredentials: getBackupCredentials
 };
 
 var assert = require('assert'),
@@ -63,7 +64,7 @@ function getAllPaged(apiConfig, page, perPage, callback) {
     });
 }
 
-function getSignedUploadUrl(apiConfig, filename, callback) {
+function getRestoreUrl(apiConfig, filename, callback) {
     assert.strictEqual(typeof apiConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -79,32 +80,9 @@ function getSignedUploadUrl(apiConfig, filename, callback) {
             Expires: 60 * 30 /* 30 minutes */
         };
 
-        var url = s3.getSignedUrl('putObject', params);
-
-        callback(null, { url : url, sessionToken: credentials.sessionToken });
-    });
-}
-
-function getSignedDownloadUrl(apiConfig, info, filename, callback) {
-    assert.strictEqual(typeof apiConfig, 'object');
-    assert.strictEqual(typeof info, 'object');
-    assert.strictEqual(typeof filename, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getBackupCredentials(apiConfig, function (error, credentials) {
-        if (error) return callback(error);
-
-        var s3 = new AWS.S3(credentials);
-
-        var params = {
-            Bucket: info.bucket,
-            Key: info.prefix + '/' + filename,
-            Expires: 60 * 30 /* 30 minutes */
-        };
-
         var url = s3.getSignedUrl('getObject', params);
 
-        callback(null, { url: url, sessionToken: credentials.sessionToken });
+        callback(null, { url: url });
     });
 }
 

src/subdomains.js

@@ -98,10 +98,10 @@ function update(subdomain, type, values, callback) {
     settings.getDnsConfig(function (error, dnsConfig) {
         if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
 
-        api(dnsConfig.provider).update(dnsConfig, config.zoneName(), subdomain, type, values, function (error) {
+        api(dnsConfig.provider).update(dnsConfig, config.zoneName(), subdomain, type, values, function (error, changeId) {
             if (error) return callback(error);
 
-            callback(null);
+            callback(null, changeId);
         });
     });
 }

src/sysinfo.js

@@ -1,19 +1,17 @@
-/* jslint node:true */
-
 'use strict';
 
-var assert = require('assert'),
-    caas = require('./sysinfo/caas.js'),
-    config = require('./config.js'),
-    ec2 = require('./sysinfo/ec2.js'),
-    util = require('util');
-
 exports = module.exports = {
     SysInfoError: SysInfoError,
 
     getIp: getIp
 };
 
+var assert = require('assert'),
+    caas = require('./sysinfo/caas.js'),
+    config = require('./config.js'),
+    ec2 = require('./sysinfo/ec2.js'),
+    util = require('util');
+
 var gCachedIp = null;
 
 function SysInfoError(reason, errorOrMessage) {

src/taskmanager.js

@@ -19,11 +19,13 @@ var appdb = require('./appdb.js'),
     cloudron = require('./cloudron.js'),
     debug = require('debug')('box:taskmanager'),
     locker = require('./locker.js'),
+    sendFailureLogs = require('./logcollector.js').sendFailureLogs,
     util = require('util'),
     _ = require('underscore');
 
 var gActiveTasks = { };
 var gPendingTasks = [ ];
+var gPlatformReady = false; // PaaS (addons) up and running
 
 var TASK_CONCURRENCY = 5;
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
@@ -39,6 +41,11 @@ function initialize(callback) {
         cloudron.events.on(cloudron.EVENT_CONFIGURED, resumeTasks);
     }
 
+    setTimeout(function () {
+        gPlatformReady = true;
+        resumeTasks();
+    }, 30000); // wait 30 seconds to signal platform ready
+
     callback();
 }
 
@@ -82,6 +89,8 @@ function resumeTasks(callback) {
         apps.forEach(function (app) {
             if (app.installationState === appdb.ISTATE_INSTALLED && app.runState === appdb.RSTATE_RUNNING) return;
 
+            if (app.installationState === appdb.ISTATE_ERROR) return;
+
             debug('Creating process for %s (%s) with state %s', app.location, app.id, app.installationState);
             startAppTask(app.id, NOOP_CALLBACK);
         });
@@ -106,6 +115,12 @@ function startAppTask(appId, callback) {
         return callback(new Error(util.format('Task for %s is already active', appId)));
     }
 
+    if (!gPlatformReady) {
+        debug('Platform not ready yet, queueing task for %s', appId);
+        gPendingTasks.push(appId);
+        return callback();
+    }
+
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug('Reached concurrency limit, queueing task for %s', appId);
         gPendingTasks.push(appId);
@@ -130,7 +145,10 @@ function startAppTask(appId, callback) {
         debug('Task for %s pid %s completed with status %s', appId, pid, code);
         if (code === null /* signal */ || (code !== 0 && code !== 50)) { // apptask crashed
             debug('Apptask crashed with code %s and signal %s', code, signal);
+            sendFailureLogs('apptask', { unit: 'box' });
             appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code + ' and signal ' + signal }, NOOP_CALLBACK);
+        } else if (code === 50) {
+            sendFailureLogs('apptask', { unit: 'box' });
         }
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task

src/test/apps-test.js

@@ -113,9 +113,9 @@ describe('Apps', function () {
             groups.create.bind(null, GROUP_1),
             groups.addMember.bind(null, groups.ADMIN_GROUP_ID, ADMIN_0.id),
             groups.addMember.bind(null, GROUP_0, USER_1.id),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, null),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, null),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, null)
         ], done);
     });
 

src/test/apptask-test.js

@@ -84,7 +84,7 @@ describe('apptask', function () {
         config.set('version', '0.5.0');
         async.series([
             database.initialize,
-            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, APP.memoryLimit),
+            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, APP.memoryLimit, null),
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
             settings.setTlsConfig.bind(null, { provider: 'caas' })
         ], done);

src/test/checkInstall

@@ -18,7 +18,6 @@ scripts=("${SOURCE_DIR}/src/scripts/rmappdir.sh" \
          "${SOURCE_DIR}/src/scripts/backupapp.sh" \
          "${SOURCE_DIR}/src/scripts/restoreapp.sh" \
          "${SOURCE_DIR}/src/scripts/reboot.sh" \
-         "${SOURCE_DIR}/src/scripts/backupswap.sh" \
          "${SOURCE_DIR}/src/scripts/collectlogs.sh" \
          "${SOURCE_DIR}/src/scripts/reloadcollectd.sh")
 

src/test/config-test.js

@@ -17,13 +17,11 @@ var config = null;
 
 describe('config', function () {
     before(function () {
-        safe.fs.unlinkSync(paths.DNS_IN_SYNC_FILE);
         delete require.cache[require.resolve('../config.js')];
         config = require('../config.js');
     });
 
     after(function () {
-        safe.fs.unlinkSync(paths.DNS_IN_SYNC_FILE);
         delete require.cache[require.resolve('../config.js')];
     });
 
@@ -32,17 +30,6 @@ describe('config', function () {
         done();
     });
 
-    it('dnsInSync() is unset', function (done) {
-        expect(config.dnsInSync()).to.not.be.ok();
-        done();
-    });
-
-    it('dnsInSync() is set', function (done) {
-        config.setDnsInSync();
-        expect(config.dnsInSync()).to.be.ok();
-        done();
-    });
-
     it('cloudron.conf generated automatically', function (done) {
         expect(fs.existsSync(path.join(config.baseDir(), 'configs/cloudron.conf'))).to.be.ok();
         done();

src/test/database-test.js

@@ -7,13 +7,15 @@
 'use strict';
 
 var appdb = require('../appdb.js'),
+    async = require('async'),
     authcodedb = require('../authcodedb.js'),
+    backupdb = require('../backupdb.js'),
     clientdb = require('../clientdb.js'),
-    hat = require('hat'),
     database = require('../database'),
     DatabaseError = require('../databaseerror.js'),
+    eventlogdb = require('../eventlogdb.js'),
     expect = require('expect.js'),
-    async = require('async'),
+    hat = require('hat'),
     settingsdb = require('../settingsdb.js'),
     tokendb = require('../tokendb.js'),
     userdb = require('../userdb.js'),
@@ -33,8 +35,8 @@ describe('database', function () {
 
     describe('userdb', function () {
         var USER_0 = {
-            id: 'uuid213',
-            username: 'uuid213',
+            id: 'uuid0',
+            username: 'uuid0',
             password: 'secret',
             email: 'safe@me.com',
             salt: 'morton',
@@ -45,29 +47,39 @@ describe('database', function () {
         };
 
         var USER_1 = {
-            id: 'uuid456',
-            username: 'uuid456',
+            id: 'uuid1',
+            username: 'uuid1',
             password: 'secret',
             email: 'safe2@me.com',
             salt: 'tata',
             createdAt: 'sometime back',
             modifiedAt: 'now',
             resetToken: '',
-            displayName: 'Herbert Heidelberg'
+            displayName: 'Herbert 1'
+        };
+
+        var USER_2 = {
+            id: 'uuid2',
+            username: 'uuid2',
+            password: 'secret',
+            email: 'safe3@me.com',
+            salt: 'tata',
+            createdAt: 'sometime back',
+            modifiedAt: 'now',
+            resetToken: '',
+            displayName: 'Herbert 2'
         };
 
         it('can add user', function (done) {
-            userdb.add(USER_0.id, USER_0, function (error) {
-                expect(!error).to.be.ok();
-                done();
-            });
+            userdb.add(USER_0.id, USER_0, done);
         });
 
         it('can add another user', function (done) {
-            userdb.add(USER_1.id, USER_1, function (error) {
-                expect(!error).to.be.ok();
-                done();
-            });
+            userdb.add(USER_1.id, USER_1, done);
+        });
+
+        it('can add another user with empty username', function (done) {
+            userdb.add(USER_2.id, USER_2, done);
         });
 
         it('cannot add same user again', function (done) {
@@ -122,13 +134,22 @@ describe('database', function () {
         it('can get all with group ids', function (done) {
             userdb.getAllWithGroupIds(function (error, all) {
                 expect(error).to.not.be.ok();
-                expect(all.length).to.equal(2);
-                var user0Copy = _.extend({}, USER_0);
-                user0Copy.groupIds = [ ];
-                expect(all[0]).to.eql(user0Copy);
-                var user1Copy = _.extend({}, USER_1);
-                user1Copy.groupIds = [ ];
-                expect(all[1]).to.eql(user1Copy);
+                expect(all.length).to.equal(3);
+
+                var userCopy;
+
+                userCopy = _.extend({}, USER_0);
+                userCopy.groupIds = [ ];
+                expect(all[0]).to.eql(userCopy);
+
+                userCopy = _.extend({}, USER_1);
+                userCopy.groupIds = [ ];
+                expect(all[1]).to.eql(userCopy);
+
+                userCopy = _.extend({}, USER_2);
+                userCopy.groupIds = [ ];
+                expect(all[2]).to.eql(userCopy);
+
                 done();
             });
         });
@@ -144,7 +165,7 @@ describe('database', function () {
         it('counts the users', function (done) {
             userdb.count(function (error, count) {
                 expect(error).to.not.be.ok();
-                expect(count).to.equal(2);
+                expect(count).to.equal(3);
                 done();
             });
         });
@@ -160,11 +181,10 @@ describe('database', function () {
             });
         });
 
-        it('cannot update with null field', function (done) {
-            userdb.update(USER_0.id, { email: null }, function (error) {
-                expect(error).to.be.ok();
-                done();
-            });
+        it('cannot update with null field', function () {
+            expect(function () {
+                userdb.update(USER_0.id, { email: null }, function () {});
+            }).to.throwError();
         });
 
         it('cannot del non-existing user', function (done) {
@@ -184,7 +204,7 @@ describe('database', function () {
 
         it('did remove the user', function (done) {
             userdb.count(function (error, count) {
-                expect(count).to.equal(1);
+                expect(count).to.equal(2);
                 done();
             });
         });
@@ -479,7 +499,8 @@ describe('database', function () {
             lastBackupId: null,
             lastBackupConfig: null,
             oldConfig: null,
-            memoryLimit: 4294967296
+            memoryLimit: 4294967296,
+            altDomain: null
         };
         var APP_1 = {
             id: 'appid-1',
@@ -498,7 +519,8 @@ describe('database', function () {
             lastBackupId: null,
             lastBackupConfig: null,
             oldConfig: null,
-            memoryLimit: 0
+            memoryLimit: 0,
+            altDomain: null
         };
 
         it('add fails due to missing arguments', function () {
@@ -515,7 +537,7 @@ describe('database', function () {
         });
 
         it('add succeeds', function (done) {
-            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, function (error) {
+            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, null, function (error) {
                 expect(error).to.be(null);
                 done();
             });
@@ -539,7 +561,7 @@ describe('database', function () {
         });
 
         it('add of same app fails', function (done) {
-            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, [ ], APP_0.accessRestriction, APP_0.memoryLimit, function (error) {
+            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, [ ], APP_0.accessRestriction, APP_0.memoryLimit, null, function (error) {
                 expect(error).to.be.a(DatabaseError);
                 expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
                 done();
@@ -611,7 +633,7 @@ describe('database', function () {
         });
 
         it('add second app succeeds', function (done) {
-            appdb.add(APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, [ ], APP_1.accessRestriction, APP_1.memoryLimit, function (error) {
+            appdb.add(APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, [ ], APP_1.accessRestriction, APP_1.memoryLimit, null, function (error) {
                 expect(error).to.be(null);
                 done();
             });
@@ -931,5 +953,172 @@ describe('database', function () {
         });
 
     });
+
+    describe('backup', function () {
+
+        it('add succeeds', function (done) {
+            var backup = {
+                id: 'backup-box',
+                version: '1.0.0',
+                type: backupdb.BACKUP_TYPE_BOX,
+                dependsOn: [ 'dep1' ]
+            };
+
+            backupdb.add(backup, function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+
+        it('get succeeds', function (done) {
+            backupdb.get('backup-box', function (error, result) {
+                expect(error).to.be(null);
+                expect(result.version).to.be('1.0.0');
+                expect(result.type).to.be(backupdb.BACKUP_TYPE_BOX);
+                expect(result.creationTime).to.be.a(Date);
+                expect(result.dependsOn).to.eql(['dep1']);
+                done();
+            });
+        });
+
+        it('get of unknown id fails', function (done) {
+            backupdb.get('somerandom', function (error, result) {
+                expect(error).to.be.a(DatabaseError);
+                expect(error.reason).to.be(DatabaseError.NOT_FOUND);
+                expect(result).to.not.be.ok();
+                done();
+            });
+        });
+
+        it('getPaged succeeds', function (done) {
+            backupdb.getPaged(1, 5, function (error, results) {
+                expect(error).to.be(null);
+                expect(results).to.be.an(Array);
+                expect(results.length).to.be(1);
+
+                expect(results[0].id).to.be('backup-box');
+                expect(results[0].version).to.be('1.0.0');
+                expect(results[0].dependsOn).to.eql(['dep1']);
+
+                done();
+            });
+        });
+
+        it('delete succeeds', function (done) {
+            backupdb.del('backup-box', function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.not.be.ok();
+
+                backupdb.get('backup-box', function (error, result) {
+                    expect(error).to.be.a(DatabaseError);
+                    expect(error.reason).to.equal(DatabaseError.NOT_FOUND);
+                    expect(result).to.not.be.ok();
+                    done();
+                });
+            });
+        });
+
+        it('add app succeeds', function (done) {
+            var backup = {
+                id: 'appbackup_appid_123',
+                version: '1.0.0',
+                type: backupdb.BACKUP_TYPE_APP,
+                dependsOn: [ ]
+            };
+
+            backupdb.add(backup, function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+
+        it('get succeeds', function (done) {
+            backupdb.get('appbackup_appid_123', function (error, result) {
+                expect(error).to.be(null);
+                expect(result.version).to.be('1.0.0');
+                expect(result.type).to.be(backupdb.BACKUP_TYPE_APP);
+                expect(result.creationTime).to.be.a(Date);
+                expect(result.dependsOn).to.eql([]);
+                done();
+            });
+        });
+
+        it('getByAppIdPaged succeeds', function (done) {
+            backupdb.getByAppIdPaged(1, 5, 'appid', function (error, results) {
+                expect(error).to.be(null);
+                expect(results).to.be.an(Array);
+                expect(results.length).to.be(1);
+
+                expect(results[0].id).to.be('appbackup_appid_123');
+                expect(results[0].version).to.be('1.0.0');
+                expect(results[0].dependsOn).to.eql([]);
+
+                done();
+            });
+        });
+
+        it('delete succeeds', function (done) {
+            backupdb.del('appbackup_appid_123', function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.not.be.ok();
+
+                backupdb.get('appbackup_appid_123', function (error, result) {
+                    expect(error).to.be.a(DatabaseError);
+                    expect(error.reason).to.equal(DatabaseError.NOT_FOUND);
+                    expect(result).to.not.be.ok();
+                    done();
+                });
+            });
+        });
+
+    });
+
+    describe('eventlog', function () {
+
+        it('add succeeds', function (done) {
+            eventlogdb.add('someid', 'some.event', { ip: '1.2.3.4' }, { appId: 'thatapp' }, function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+
+        it('get succeeds', function (done) {
+            eventlogdb.get('someid', function (error, result) {
+                expect(error).to.be(null);
+                expect(result.id).to.be('someid');
+                expect(result.action).to.be('some.event');
+                expect(result.creationTime).to.be.a(Date);
+
+                expect(result.source).to.be.eql({ ip: '1.2.3.4' });
+                expect(result.data).to.be.eql({ appId: 'thatapp' });
+
+                done();
+            });
+        });
+
+        it('get of unknown id fails', function (done) {
+            eventlogdb.get('notfoundid', function (error, result) {
+                expect(error).to.be.a(DatabaseError);
+                expect(error.reason).to.be(DatabaseError.NOT_FOUND);
+                expect(result).to.not.be.ok();
+                done();
+            });
+        });
+
+        it('getAllPaged succeeds', function (done) {
+            eventlogdb.getAllPaged(1, 1, function (error, results) {
+                expect(error).to.be(null);
+                expect(results).to.be.an(Array);
+                expect(results.length).to.be(1);
+
+                expect(results[0].id).to.be('someid');
+                expect(results[0].action).to.be('some.event');
+                expect(results[0].source).to.be.eql({ ip: '1.2.3.4' });
+                expect(results[0].data).to.be.eql({ appId: 'thatapp' });
+
+                done();
+            });
+        });
+    });
 });
 

src/test/eventlog-test.js

@@ -0,0 +1,81 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var database = require('../database.js'),
+    expect = require('expect.js'),
+    eventlog = require('../eventlog.js'),
+    EventLogError = eventlog.EventLogError;
+
+function setup(done) {
+    // ensure data/config/mount paths
+    database.initialize(function (error) {
+        expect(error).to.be(null);
+        done();
+    });
+}
+
+function cleanup(done) {
+    database._clear(done);
+}
+
+describe('Eventlog', function () {
+    before(setup);
+    after(cleanup);
+
+    var eventId;
+
+    it('add succeeds', function (done) {
+        eventlog.add('some.event', { ip: '1.2.3.4' }, { appId: 'thatapp' }, function (error, result) {
+            expect(error).to.be(null);
+            expect(result.id).to.be.ok();
+
+            eventId = result.id;
+
+            done();
+        });
+    });
+
+    it('get succeeds', function (done) {
+        eventlog.get(eventId, function (error, result) {
+            expect(error).to.be(null);
+            expect(result.id).to.be(eventId);
+            expect(result.action).to.be('some.event');
+            expect(result.creationTime).to.be.a(Date);
+
+            expect(result.source).to.be.eql({ ip: '1.2.3.4' });
+            expect(result.data).to.be.eql({ appId: 'thatapp' });
+
+            done();
+        });
+    });
+
+    it('get of unknown id fails', function (done) {
+        eventlog.get('notfoundid', function (error, result) {
+            expect(error).to.be.a(EventLogError);
+            expect(error.reason).to.be(EventLogError.NOT_FOUND);
+            expect(result).to.not.be.ok();
+
+            done();
+        });
+    });
+
+    it('getAllPaged succeeds', function (done) {
+        eventlog.getAllPaged(1, 1, function (error, results) {
+            expect(error).to.be(null);
+            expect(results).to.be.an(Array);
+            expect(results.length).to.be(1);
+
+            expect(results[0].id).to.be(eventId);
+            expect(results[0].action).to.be('some.event');
+            expect(results[0].source).to.be.eql({ ip: '1.2.3.4' });
+            expect(results[0].data).to.be.eql({ appId: 'thatapp' });
+
+            done();
+        });
+    });
+});