up link is relative

For custom domains, we do not set the A record for the naked domain
(because the user might be using it for his own). This means that
a:domain.com will not work.

The solution is to simply use the admin domain.

CHANGES

@@ -372,3 +372,70 @@
 - Fix `cloudron exec` container to have same namespaces as app
 - Add developmentMode to manifest
 
+[0.6.3]
+- Make sending invite for new users optional
+
+[0.6.4]
+- Add support for display names
+- Send invite links to admins for user setup
+- Enforce stronger passwords
+
+[0.6.5]
+- Finalize stronger password requirement
+
+[0.7.0]
+- Upgrade to 15.10
+- Do not remove docker images when in use by another container
+- Fix sporadic error when reconfiguring apps
+- Handle journald crashes gracefully
+
+[0.7.1]
+- Allow admins to edit users
+- Fix graphs
+- Support more LDAP cases
+- Allow appstore deep linking
+
+[0.7.2]
+- Fix 5xx errors when password does not meet requirements
+- Improved box update management using prereleases
+- Less aggressive disk space checks
+
+[0.8.0]
+- MySQL addon : multiple database support
+
+[0.8.1]
+- Set Host HTTP header when querying healthCheckPath
+- Show application Changelog in app update emails
+
+[0.9.0]
+- Fix bug in multdb mysql addon backup
+- Add initial user group support
+- Improved app memory limit handling
+
+[0.9.1]
+- Introduce per app group access control
+
+[0.9.2]
+- Fix bug where reconfiguring apps would trigger memory limit warning
+- Allow more apps to be installed in bigger sized cloudrons
+- Allow user to override memory limit warning and install anyway
+
+[0.9.3]
+- Admin flag is handled outside of groups
+- User interface fixes for groups
+- Allow to set access restrictions on app installation
+
+[0.10.0]
+- Upgrade to docker 1.10.2
+- Fix MySQL addon to handle heavier loads
+- Allow listing and download of backups (using the CLI tool)
+- Ubuntu security updates till 8th March 2016 (http://www.ubuntu.com/usn)
+
+[0.10.1]
+- Fix Let's Encrypt certificate renewal
+
+[0.10.2]
+- Apps can now bind with username or email with LDAP
+- Disallow updating an app with mismatching manifest id
+- Use admin domain instead of naked domain in the SPF record
+- Download Lets Encrypt intermediate cert

README.md

@@ -11,6 +11,7 @@ The smart server currently relies on an AWS account with access to Route53 and S
 First create a virtual private server with Ubuntu 15.04 and run the following commands in an ssh session to initialize the base image:
 
 ```
-TODO curl from a well known released version of installer.sh
+curl https://s3.amazonaws.com/prod-cloudron-releases/installer.sh -o installer.sh
+chmod +x installer.sh
 ./installer.sh <domain> <aws access key> <aws acccess secret> <backup bucket> <provider> <release sha1>
 ```

baseimage/createImage

@@ -150,12 +150,6 @@ if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${
     exit 1
 fi
 
-echo "Copy over certs"
-cd "${SCRIPT_DIR}/../../secrets"
-blackbox_cat installer/server.crt.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/server.crt"
-blackbox_cat installer/server.key.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/server.key"
-blackbox_cat installer_ca/ca.crt.gpg  | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/ca.crt"
-
 echo "Shutting down server with id : ${server_id}"
 $ssh202 "root@${server_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail
 

baseimage/digitalocean.sh

@@ -30,7 +30,7 @@ function create_droplet() {
     local box_name="$2"
 
     local image_region="sfo1"
-    local ubuntu_image_slug="ubuntu-15-04-x64" # id=12658446
+    local ubuntu_image_slug="ubuntu-15-10-x64"
     local box_size="512mb"
 
     local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"

baseimage/initializeBaseUbuntuImage.sh

@@ -12,6 +12,13 @@ readonly USER_DATA_DIR="/home/yellowtent/data"
 
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+function die {
+    echo $1
+    exit 1
+}
+
+[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
+
 if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
     source "${SOURCE_DIR}/INFRA_VERSION"
 else
@@ -42,7 +49,7 @@ export DEBIAN_FRONTEND=noninteractive
 
 echo "=== Upgrade ==="
 apt-get update
-apt-get upgrade -y
+apt-get dist-upgrade -y
 apt-get install -y curl
 
 # Setup firewall before everything. docker creates it's own chain and the -X below will remove it
@@ -87,7 +94,8 @@ apt-get -y install btrfs-tools
 
 echo "==== Install docker ===="
 # install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
-curl https://get.docker.com/builds/Linux/x86_64/docker-1.9.1 > /usr/bin/docker
+curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
+apt-get -y install aufs-tools
 chmod +x /usr/bin/docker
 groupadd docker
 cat > /etc/systemd/system/docker.socket <<EOF
@@ -121,11 +129,11 @@ LimitCORE=infinity
 WantedBy=multi-user.target
 EOF
 
-echo "=== Setup btrfs docker data ==="
-fallocate -l "8192m" "${USER_DATA_FILE}" # 8gb start
+echo "=== Setup btrfs data ==="
+truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
 mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
-echo "${USER_DATA_FILE} ${USER_DATA_DIR} btrfs loop,nosuid 0 0" >> /etc/fstab
-mkdir -p "${USER_DATA_DIR}" && mount "${USER_DATA_FILE}"
+mkdir -p "${USER_DATA_DIR}"
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
 
 systemctl daemon-reload
 systemctl enable docker
@@ -175,15 +183,16 @@ fi
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
+[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
 
 echo "==== Install build-essential ===="
 apt-get -y install build-essential rcconf
 
-
 echo "==== Install mysql ===="
 debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
 debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
 apt-get -y install mysql-server
+[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
 
 echo "==== Install pwgen ===="
 apt-get -y install pwgen
@@ -208,6 +217,7 @@ curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf
 ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
 ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
 apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
 echo "=== Rebuilding npm packages ==="
 cd "${INSTALLER_SOURCE_DIR}" && npm install --production
@@ -222,12 +232,15 @@ fi
 cat > /etc/systemd/system/cloudron-installer.service <<EOF
 [Unit]
 Description=Cloudron Installer
+; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
+BindsTo=systemd-journald.service
 
 [Service]
 Type=idle
 ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
 Environment="DEBUG=installer*,connect-lastmile" ${provisionEnv}
-KillMode=process
+; kill any child (installer.sh) as well
+KillMode=control-group
 Restart=on-failure
 
 [Install]
@@ -256,7 +269,7 @@ echo "==== Install box-setup systemd script ===="
 cat > /etc/systemd/system/box-setup.service <<EOF
 [Unit]
 Description=Box Setup
-Before=docker.service
+Before=docker.service collectd.service mysql.service
 After=do-resize.service
 
 [Service]
@@ -278,6 +291,11 @@ sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
     -i /etc/systemd/journald.conf
 
+# When rotating logs, systemd kills journald too soon sometimes
+# See https://github.com/systemd/systemd/issues/1353 (this is upstream default)
+sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
+    -i /lib/systemd/system/systemd-journald.service
+
 sync
 
 # Configure time

gulpfile.js

@@ -10,7 +10,7 @@ var ejs = require('gulp-ejs'),
     serve = require('gulp-serve'),
     sass = require('gulp-sass'),
     sourcemaps = require('gulp-sourcemaps'),
-    minifyCSS = require('gulp-minify-css'),
+    cssnano = require('gulp-cssnano'),
     autoprefixer = require('gulp-autoprefixer'),
     argv = require('yargs').argv;
 
@@ -39,7 +39,7 @@ gulp.task('3rdparty', function () {
 // JavaScript
 // --------------
 
-gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
+gulp.task('js', ['js-index', 'js-setup', 'js-update'], function () {});
 
 var oauth = {
     clientId: argv.clientId || 'cid-webadmin',
@@ -80,14 +80,6 @@ gulp.task('js-setup', function () {
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('js-error', function () {
-    gulp.src(['webadmin/src/js/error.js'])
-        .pipe(sourcemaps.init())
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'));
-});
-
 gulp.task('js-update', function () {
     gulp.src(['webadmin/src/js/update.js'])
         .pipe(sourcemaps.init())
@@ -127,7 +119,7 @@ gulp.task('css', function () {
         .pipe(sourcemaps.init())
         .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
         .pipe(autoprefixer())
-        .pipe(minifyCSS())
+        .pipe(cssnano())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist'))
         .pipe(gulp.dest('setup/splash/website'));
@@ -149,7 +141,6 @@ gulp.task('watch', ['default'], function () {
     gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
     gulp.watch(['webadmin/src/templates/*.html'], ['html-templates']);
     gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
-    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
     gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
     gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
     gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);

installer.sh.ejs

@@ -20,10 +20,15 @@ readonly provider="${5}"
 readonly revision="${6}"
 
 # environment specific urls
-readonly api_server_origin="https://api.dev.cloudron.io"
-readonly web_server_origin="https://dev.cloudron.io"
-readonly release_bucket_url="https://s3.amazonaws.com/dev-cloudron-releases"
-readonly versions_url="https://s3.amazonaws.com/dev-cloudron-releases/versions.json"
+<% if (env === 'prod') { %>
+readonly api_server_origin="https://api.cloudron.io"
+readonly web_server_origin="https://cloudron.io"
+<% } else { %>
+readonly api_server_origin="https://api.<%= env %>.cloudron.io"
+readonly web_server_origin="https://<%= env %>.cloudron.io"
+<% } %>
+readonly release_bucket_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases"
+readonly versions_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases/versions.json"
 readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
 
 # runtime consts
@@ -132,7 +137,7 @@ cat > /root/provision.json <<EOF
             "secretAccessKey": "${aws_access_key_secret}"
         },
         "tlsConfig": {
-            "provider": "letsencrypt-dev"
+            "provider": "letsencrypt-<%= env %>"
         }
     }
 }

installer/src/installer.js

@@ -15,13 +15,11 @@ exports = module.exports = {
     InstallerError: InstallerError,
 
     provision: provision,
-    retire: retire,
 
     _ensureVersion: ensureVersion
 };
 
 var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'),
-    RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh'),
     SUDO = '/usr/bin/sudo';
 
 function InstallerError(reason, info) {
@@ -36,6 +34,7 @@ util.inherits(InstallerError, Error);
 InstallerError.INTERNAL_ERROR = 1;
 InstallerError.ALREADY_PROVISIONED = 2;
 
+// system until file has KillMode=control-group to bring down child processes
 function spawn(tag, cmd, args, callback) {
     assert.strictEqual(typeof tag, 'string');
     assert.strictEqual(typeof cmd, 'string');
@@ -62,21 +61,6 @@ function spawn(tag, cmd, args, callback) {
     });
 }
 
-function retire(args, callback) {
-    assert.strictEqual(typeof args, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var pargs = [ RETIRE_CMD ];
-    pargs.push('--data', JSON.stringify(args.data));
-
-    debug('retire: calling with args %j', pargs);
-
-    if (process.env.NODE_ENV === 'test') return callback(null);
-
-    // sudo is required for retire()
-    spawn('retire', SUDO, pargs, callback);
-}
-
 function ensureVersion(args, callback) {
     assert.strictEqual(typeof args, 'object');
     assert.strictEqual(typeof callback, 'function');

installer/src/scripts/installer.sh

@@ -7,7 +7,7 @@ readonly DATA_DIR=/home/yellowtent/data
 
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly json="${script_dir}/../../node_modules/.bin/json"
-readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 180"
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
 
 readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
 

installer/src/scripts/retire.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-# This script is called once at the end of a cloudrons lifetime
-
-set -eu -o pipefail
-
-readonly BOX_SRC_DIR=/home/yellowtent/box
-
-arg_data=""
-
-args=$(getopt -o "" -l "data:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --data) arg_data="$2";;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-
-    shift 2
-done
-
-echo "Setting up splash screen"
-"${BOX_SRC_DIR}/setup/splashpage.sh" --retire --data "${arg_data}" # show splash
-"${BOX_SRC_DIR}/setup/stop.sh" # stop the cloudron code
-
-systemctl stop docker # stop the apps
-systemctl stop cloudron-installer # stop the installer
-

installer/src/server.js

@@ -11,13 +11,11 @@ var assert = require('assert'),
     fs = require('fs'),
     http = require('http'),
     HttpError = require('connect-lastmile').HttpError,
-    https = require('https'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     installer = require('./installer.js'),
     json = require('body-parser').json,
     lastMile = require('connect-lastmile'),
     morgan = require('morgan'),
-    path = require('path'),
     superagent = require('superagent');
 
 exports = module.exports = {
@@ -28,8 +26,7 @@ exports = module.exports = {
 var PROVISION_CONFIG_FILE = '/root/provision.json';
 var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
 
-var gHttpsServer = null, // provision server; used for install/restore
-    gHttpServer = null; // update server; used for updates
+var gHttpServer = null; // update server; used for updates
 
 function provisionDigitalOcean(callback) {
     if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
@@ -74,23 +71,6 @@ function update(req, res, next) {
     next(new HttpSuccess(202, { }));
 }
 
-function retire(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
-
-    if (typeof req.body.data.tlsCert !== 'string') console.error('No TLS cert provided');
-    if (typeof req.body.data.tlsKey !== 'string') console.error('No TLS key provided');
-
-    debug('retire: received from appstore %j', req.body);
-
-    installer.retire(req.body, function (error) {
-        if (error) console.error(error);
-    });
-
-    next(new HttpSuccess(202, {}));
-}
-
 function startUpdateServer(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -114,53 +94,6 @@ function startUpdateServer(callback) {
     gHttpServer.listen(2020, '127.0.0.1', callback);
 }
 
-function startProvisionServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Starting provision server');
-
-    var app = express();
-
-    var router = new express.Router();
-
-    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
-
-    app.use(json({ strict: true }))
-       .use(router)
-       .use(lastMile());
-
-    router.post('/api/v1/installer/retire', retire);
-
-    var caPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? 'test/certs' : 'certs');
-    var certPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? 'test/certs' : 'certs');
-
-    var options = {
-        key: fs.readFileSync(path.join(certPath, 'server.key')),
-        cert: fs.readFileSync(path.join(certPath, 'server.crt')),
-        ca: fs.readFileSync(path.join(caPath, 'ca.crt')),
-
-        // request cert from client and only allow from our CA
-        requestCert: true,
-        rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0' // this is set in the tests
-    };
-
-    gHttpsServer = https.createServer(options, app);
-    gHttpsServer.on('error', console.error);
-
-    gHttpsServer.listen(process.env.NODE_ENV === 'test' ? 4443 : 886, '0.0.0.0', callback);
-}
-
-function stopProvisionServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Stopping provision server');
-
-    if (!gHttpsServer) return callback(null);
-
-    gHttpsServer.close(callback);
-    gHttpsServer = null;
-}
-
 function stopUpdateServer(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -189,7 +122,6 @@ function start(callback) {
 
         actions = [
             startUpdateServer,
-            startProvisionServer,
             provisionDigitalOcean
         ];
     }
@@ -201,8 +133,7 @@ function stop(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     async.series([
-        stopUpdateServer,
-        stopProvisionServer
+        stopUpdateServer
     ], callback);
 }
 

installer/src/test/certs/ca.crt

@@ -1,24 +0,0 @@
------BEGIN CERTIFICATE-----
-MIID9zCCAt+gAwIBAgIJAMPL81PAySGAMA0GCSqGSIb3DQEBBQUAMFoxCzAJBgNV
-BAYTAlVTMQswCQYDVQQIEwJDQTELMAkGA1UEBxMCU0MxFTATBgNVBAoTDENsb3Vk
-cm9uIEluYzEaMBgGA1UEAxMRSW5zdGFsbCBTZXJ2ZXIgQ0EwHhcNMTUwMTE2MDEy
-NDM2WhcNMTYwMTE2MDEyNDM2WjBaMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0Ex
-CzAJBgNVBAcTAlNDMRUwEwYDVQQKEwxDbG91ZHJvbiBJbmMxGjAYBgNVBAMTEUlu
-c3RhbGwgU2VydmVyIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
-31TkOEC3JXtieHiZgM5qWw771rV2JEDKs1C68+n/OmKrp3zAQV08A+w/KVurn1P9
-gZlYF+CBRVZDV8lYbWzc6PgMPWEDHHV72FS5Kq6ZyikB+r5OQJ8qU61y840h6ZCD
-MEYr6N9qXm9wSApJBQ/key/pg7+95B2CFYRrg5NVstIYqpJ1lyxCMFTrjYAmteOB
-Bi/4GPApu9Tj0ifTMbZFGTPtWm/yhCZ6Anm6w+ok9tDMpPC6kRgUJ3B4HY75D9dV
-aWSls9jdZw4JU1jIFlAdUjhGEEmHWOzAD8vBjvuBqcf9NQwvieWG5tDYfZ6DYRC2
-/aG1C5UWhFLDv2/F+56k3wIDAQABo4G/MIG8MB0GA1UdDgQWBBQ088hd2sIIqVtw
-xJeAkCORdclFRjCBjAYDVR0jBIGEMIGBgBQ088hd2sIIqVtwxJeAkCORdclFRqFe
-pFwwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQswCQYDVQQHEwJTQzEVMBMG
-A1UEChMMQ2xvdWRyb24gSW5jMRowGAYDVQQDExFJbnN0YWxsIFNlcnZlciBDQYIJ
-AMPL81PAySGAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAJcW+Wmz
-/o0JBC2WsMjUjxVrzOiu9bdKQ1yn83Zcv74zEfmWfJotVOK1oKsTyOZfTvvWrpLc
-GXXhh4oXWsNnFII3uJyZIY3v/DoE0pa7TCZhLYFbL2kEaC5rTwe/+VScHy5ROOiu
-+gnzOU3MyrcMTT0v4qcT0NlkIptRdvIYNpqfXO6vG9sMp4C/NwWhl/IfHkIAv0eH
-l3HTr8wxgldCjxbnJgYkyUcWAmLi2YEXKCEPWmsfqp3Z+Ng1M+A9OKjJLHWowl9X
-4arvn6WaUbZjRxxjvK199If1R6KWwD6YQ9cKH4Ex4/hhIqg5I3MQFu+pOq/b0XH/
-9I10o6FVU7vcFkQ=
------END CERTIFICATE-----

installer/src/test/certs/server.crt

@@ -1,20 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDMDCCAhgCCQCDr1HQJBr1izANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJV
-UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNDMREwDwYDVQQKDAhDbG91ZHJvbjEe
-MBwGA1UEAwwVaW5zdGFsbGVyLmNsb3Vkcm9uLmlvMB4XDTE1MTExNjIzMTcwMloX
-DTE2MTExNTIzMTcwMlowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQswCQYD
-VQQHDAJTQzERMA8GA1UECgwIQ2xvdWRyb24xHjAcBgNVBAMMFWluc3RhbGxlci5j
-bG91ZHJvbi5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0suQX7
-hKBhYsSH0msnEPVbRDIotYbtVDav/v7Sb/fRU7qVoL31tj2iZRDJRJ27uRM3J4ye
-6hgJAAwQGtfXrcVZY3SOAlGXsFZF0wgBCw0pGtgF3HA1BcwbCwAd06J6w3lKActA
-DMEUio/jRXpYELUU2Nzopq0MsMyyBSBkNC18i0HUB8vkF8yQvb1OpbcxERbpf3D5
-zjeFf5kIE/k8lwBz1vMF0uAA2GfcXxs3dyDaxVteWeevVYZzAoY9EcUyBWX7OQnx
-aUygl3OywN+xOJKXKCQpckzDvr9Vp1sKItoMMy5y81SyNhZIMBYGGG+oNp/wSgQf
-Cht+LupI+bXoYrMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAgPHZx52qYuEUdzVO
-t/+VXO7dxJkONYU8sjTYIfJme8ZZd7beZBMUni5s2gvv6i5HFyJ2Ol88sv8hAaI/
-6Vmbszml+5tLyPK8Gygk62l6OcKDwU/yazTxxCApulNy1SV34kzruXUMZ28ybcqA
-XJywMMx4RDmSIBXPdDCeaOgYwI7Wk56obJ8sa2+Z6100GNoX+qBSOsWMMJW+ohnp
-eQWHkTOJzU4hIMfZCbW0cF5Xn/35xEh0xxaH7XWglJLM9neBPba+Ydz7567mN9co
-vgv2dE5ZOKSjG63CtUvv819dvbWVKq8jiMCqPGRcr1iSeqbC02tnx0W762980uSx
-QfOgAw==
------END CERTIFICATE-----

installer/src/test/certs/server.key

@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEArSy5BfuEoGFixIfSaycQ9VtEMii1hu1UNq/+/tJv99FTupWg
-vfW2PaJlEMlEnbu5EzcnjJ7qGAkADBAa19etxVljdI4CUZewVkXTCAELDSka2AXc
-cDUFzBsLAB3TonrDeUoBy0AMwRSKj+NFelgQtRTY3OimrQywzLIFIGQ0LXyLQdQH
-y+QXzJC9vU6ltzERFul/cPnON4V/mQgT+TyXAHPW8wXS4ADYZ9xfGzd3INrFW15Z
-569VhnMChj0RxTIFZfs5CfFpTKCXc7LA37E4kpcoJClyTMO+v1WnWwoi2gwzLnLz
-VLI2FkgwFgYYb6g2n/BKBB8KG34u6kj5tehiswIDAQABAoIBAGNAQ5bbLYsh5ZKP
-6ZhCHqUQtsgsrsVzFhX1zqbLgyK8VUmV4jedMOKoRVZWlD32zj7mGIOuvKoj1mQT
-gt78HPsDnU266jdLQeRgRm/K8UOMsHbo/QtOSFFPmoFpltcDly7XrKmJvwWWOUf4
-UOSqvoCaPyR1Lrn1kQrwaKHE7Ga4jfyOrIq9JI7y/ih+Y7D8xcMnyLAsjyVkSAtr
-+XrGNHcx3yPuBmjaOglzeb6Ksdpt4ETElrvH3ByT5EV2zUVr9Txv+m8xSVBZfea9
-aE7lWSQoOUz+e6RhIX3Df/QfR6KkDblAwEF9Se98DWcz46Y34oc2E0lSoJYpoPxP
-vbRlfDkCgYEA3nAc8kDRkbQObSfnVjpijBSP5hfr3jX+XTbxK7Y3aTMViY+87iWK
-bLNuX+2JRCmRjk0wy2YXnJQV3sU/EO5gLhOz9060MIHgFISq4KRgPorN/EFWryOe
-mDzhPIuhZLMetv0ajS3Z5IxIAs+FLu7Yx9em80q540UA3kXsFWe2lpUCgYEAx03E
-kk5zLirVFtoyP/yAES+KVppqBweCUA5vVxB8H26oIhi8G8kT4b77x6wXxQzdsA4H
-a4ou3ZBZVK41PREgG1MWgzpbwk49T1FX6TLtvdhr/9QhYC+RIynynA/pA36LSKT5
-pvWegYB4+9jaPrQ5L1zcrLF2XlTsgpuC43kXKicCgYA0dXxeJatHEY/VbnPAgkR7
-hN3rBfk6jsFOeoamKHMo/EM4Dg4gm/npaOe+9+ZHjQYm6U14qrsm0kXWI+6br5w/
-QaZPzN/yEK8oJ6GlGR8ZoOKzezVWWLAudy0neka12QiFX2vDn+yjWfIht49RYkL9
-3n4hIp50WvG5egQTiEIngQKBgCn9yJzKypm/jIX0EwJIQPNeANeeURiKDHqxj+PY
-JU66EdKdQ4TXKMk3Y/T93UQ3Ib4mNooB4z3rW+brjWwAX7NiHiwn741QzroXeV44
-zL5jCt4r45xQaVPvUp5u+7kwwEfd+nui5HKEjvkBB3qOnj3MYvI/saDOY8Zg3YLv
-0GGhAoGANBwFcDgwP9KDt0NxKXhe3rlSUyfGSSUF89hZPrLDCiaGFURD/w4j3EGr
-Ui9Rcwm2ymqlFzTO4JYKy1/pRCWA7GDfslICJPOPG3Wytsjog0WymQuMjYC2tL/+
-RwD0qG0/aBGE4PbigPRoJ/7BGZLKtdy99P0wyFC3o6OBoAl3Zqo=
------END RSA PRIVATE KEY-----

installer/src/test/installer-test.js

@@ -109,46 +109,6 @@ describe('Server', function () {
         });
     });
 
-    describe('retire', function () {
-        var data = {
-            data: {
-                tlsKey: 'key',
-                tlsCert: 'cert'
-            }
-        };
-
-        before(function (done) {
-            process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead
-            server.start(done);
-        });
-
-        after(function (done) {
-            server.stop(done);
-            delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
-        });
-
-        Object.keys(data).forEach(function (key) {
-            it('fails due to missing ' + key, function (done) {
-                var dataCopy = _.merge({ }, data);
-                delete dataCopy[key];
-
-                request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(dataCopy).end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-        });
-
-        it('succeeds', function (done) {
-            request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(data).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(202);
-                done();
-            });
-        });
-    });
-
     describe('ensureVersion', function () {
         before(function () {
             process.env.NODE_ENV = undefined;

installer/systemd/box-setup.sh

@@ -19,12 +19,11 @@ fi
 
 # all sizes are in mb
 readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
-readonly swap_size="${physical_memory}"
+readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
 readonly app_count=$((${physical_memory} / 200)) # estimated app count
 readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
 readonly disk_size=$((disk_size_gb * 1024))
 readonly backup_swap_size=1024
-# readonly system_size=5120 # 5 gigs for system libs, installer, box code and tmp
 readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
 readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
 
@@ -58,8 +57,10 @@ fi
 echo "Resizing data volume"
 home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
 echo "Resizing up btrfs user data to size ${home_data_size}M"
-umount "${USER_DATA_DIR}"
-fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
-mount "${USER_DATA_FILE}"
+umount "${USER_DATA_DIR}" || true
+# Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data
+# fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
+truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
 btrfs filesystem resize max "${USER_DATA_DIR}"
 

migrations/20160119113143-users-add-displayName.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN displayName VARCHAR(512) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN displayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160205135326-apps-add-memoryLimit.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN memoryLimit BIGINT DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN memoryLimit', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208031241-groups-add-table.js

@@ -0,0 +1,21 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE groups(" +
+                "id VARCHAR(128) NOT NULL UNIQUE," +
+                "name VARCHAR(128) NOT NULL UNIQUE," +
+                "PRIMARY KEY(id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208100000-groupMembers-add-table.js

@@ -0,0 +1,22 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = "CREATE TABLE IF NOT EXISTS groupMembers(" +
+    			"groupId VARCHAR(128) NOT NULL," +
+    			"userId VARCHAR(128) NOT NULL," +
+    			"FOREIGN KEY(groupId) REFERENCES groups(id)," +
+    			"FOREIGN KEY(userId) REFERENCES users(id));";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groupMembers', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208164735-groups-add-admin.js

@@ -0,0 +1,30 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var async = require('async');
+
+var ADMIN_GROUP_ID = 'admin'; // see groups.js
+
+exports.up = function(db, callback) {
+	async.series([
+		db.runSql.bind(db, 'START TRANSACTION;'),
+		db.runSql.bind(db, 'INSERT INTO groups (id, name) VALUES (?, ?)', [ ADMIN_GROUP_ID, 'admin' ]),
+		function migrateAdminFlag(done) {
+			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
+				if (error) return done(error);
+
+				console.dir(results);
+
+				async.eachSeries(results, function (r, next) {
+					db.runSql('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ ADMIN_GROUP_ID, r.id ], next);
+				}, done);
+			});
+		},
+		db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+		db.runSql.bind(db, 'COMMIT')
+	], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20160307172229-backups-add-table.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE backups(" +
+            "filename VARCHAR(128) NOT NULL," +
+            "creationTime TIMESTAMP," +
+            "version VARCHAR(128) NOT NULL," +
+            "type VARCHAR(16) NOT NULL," +
+            "dependsOn VARCHAR(4096)," +
+            "state VARCHAR(16) NOT NULL," +
+            "PRIMARY KEY (filename))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE backups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -18,8 +18,20 @@ CREATE TABLE IF NOT EXISTS users(
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
+    displayName VARCHAR(512) DEFAULT '',
     PRIMARY KEY(id));
 
+CREATE TABLE IF NOT EXISTS groups(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS groupMembers(
+    groupId VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    FOREIGN KEY(groupId) REFERENCES groups(id),
+    FOREIGN KEY(userId) REFERENCES users(id));
+
 CREATE TABLE IF NOT EXISTS tokens(
     accessToken VARCHAR(128) NOT NULL UNIQUE,
     identifier VARCHAR(128) NOT NULL,
@@ -52,6 +64,7 @@ CREATE TABLE IF NOT EXISTS apps(
     accessRestrictionJson TEXT,
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    memoryLimit BIGINT DEFAULT 0,
 
     lastBackupId VARCHAR(128),
     lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
@@ -85,3 +98,12 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     value VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
+CREATE TABLE IF NOT EXISTS backups(
+    filename VARCHAR(128) NOT NULL, /* s3 url */
+    creationTime TIMESTAMP,
+    version VARCHAR(128) NOT NULL, /* app version or box version */
+    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
+    dependsOn VARCHAR(4096), /* comma separate list of objects this backup depends on */
+    state VARCHAR(16) NOT NULL,
+
+    PRIMARY KEY (filename));

npm-shrinkwrap.json

@@ -3,9 +3,9 @@
   "version": "0.0.1",
   "dependencies": {
     "async": {
-      "version": "1.5.0",
-      "from": "https://registry.npmjs.org/async/-/async-1.5.0.tgz",
-      "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz"
+      "version": "1.5.2",
+      "from": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
     },
     "attempt": {
       "version": "1.0.1",
@@ -13,37 +13,39 @@
       "resolved": "http://registry.npmjs.org/attempt/-/attempt-1.0.1.tgz"
     },
     "aws-sdk": {
-      "version": "2.2.22",
-      "from": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.2.22.tgz",
-      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.2.22.tgz",
+      "version": "2.2.35",
+      "from": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.2.35.tgz",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.2.35.tgz",
       "dependencies": {
         "sax": {
-          "version": "0.5.3",
-          "from": "https://registry.npmjs.org/sax/-/sax-0.5.3.tgz",
-          "resolved": "https://registry.npmjs.org/sax/-/sax-0.5.3.tgz"
+          "version": "1.1.5",
+          "from": "https://registry.npmjs.org/sax/-/sax-1.1.5.tgz",
+          "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.5.tgz"
         },
         "xml2js": {
-          "version": "0.2.8",
-          "from": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz",
-          "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz"
+          "version": "0.4.16",
+          "from": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.16.tgz",
+          "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.16.tgz"
         },
         "xmlbuilder": {
-          "version": "0.4.2",
-          "from": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz",
-          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz"
+          "version": "4.2.1",
+          "from": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz",
+          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz",
+          "dependencies": {
+            "lodash": {
+              "version": "4.3.0",
+              "from": "https://registry.npmjs.org/lodash/-/lodash-4.3.0.tgz",
+              "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.3.0.tgz"
+            }
+          }
         }
       }
     },
     "body-parser": {
-      "version": "1.14.1",
-      "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
+      "version": "1.15.0",
+      "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.15.0.tgz",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.15.0.tgz",
       "dependencies": {
-        "bytes": {
-          "version": "2.1.0",
-          "from": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz",
-          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz"
-        },
         "content-type": {
           "version": "1.0.1",
           "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz",
@@ -55,13 +57,13 @@
           "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
         },
         "http-errors": {
-          "version": "1.3.1",
-          "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+          "version": "1.4.0",
+          "from": "https://registry.npmjs.org/http-errors/-/http-errors-1.4.0.tgz",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.4.0.tgz",
           "dependencies": {
             "inherits": {
               "version": "2.0.1",
-              "from": "inherits@>=2.0.1 <2.1.0",
+              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
               "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
             },
             "statuses": {
@@ -72,9 +74,9 @@
           }
         },
         "iconv-lite": {
-          "version": "0.4.12",
-          "from": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz"
+          "version": "0.4.13",
+          "from": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.13.tgz",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.13.tgz"
         },
         "on-finished": {
           "version": "2.3.0",
@@ -89,25 +91,15 @@
           }
         },
         "qs": {
-          "version": "5.1.0",
-          "from": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz"
+          "version": "6.1.0",
+          "from": "https://registry.npmjs.org/qs/-/qs-6.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.1.0.tgz"
         },
         "raw-body": {
           "version": "2.1.5",
           "from": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.5.tgz",
           "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.5.tgz",
           "dependencies": {
-            "bytes": {
-              "version": "2.2.0",
-              "from": "https://registry.npmjs.org/bytes/-/bytes-2.2.0.tgz",
-              "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.2.0.tgz"
-            },
-            "iconv-lite": {
-              "version": "0.4.13",
-              "from": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.13.tgz",
-              "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.13.tgz"
-            },
             "unpipe": {
               "version": "1.0.0",
               "from": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -116,24 +108,24 @@
           }
         },
         "type-is": {
-          "version": "1.6.10",
-          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.10.tgz",
-          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.10.tgz",
+          "version": "1.6.11",
+          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz",
           "dependencies": {
             "media-typer": {
               "version": "0.3.0",
-              "from": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-              "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+              "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
             },
             "mime-types": {
-              "version": "2.1.8",
-              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.8.tgz",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.8.tgz",
+              "version": "2.1.9",
+              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
               "dependencies": {
                 "mime-db": {
-                  "version": "1.20.0",
-                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.20.0.tgz",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.20.0.tgz"
+                  "version": "1.21.0",
+                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
                 }
               }
             }
@@ -147,28 +139,28 @@
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.2.0.tgz"
     },
     "cloudron-manifestformat": {
-      "version": "2.2.0",
-      "from": "cloudron-manifestformat@2.2.0",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-2.2.0.tgz",
+      "version": "2.3.0",
+      "from": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-2.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-2.3.0.tgz",
       "dependencies": {
         "java-packagename-regex": {
           "version": "1.0.0",
-          "from": "java-packagename-regex@>=1.0.0 <2.0.0",
+          "from": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz",
           "resolved": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz"
         },
         "safetydance": {
           "version": "0.0.15",
-          "from": "safetydance@0.0.15",
+          "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz",
           "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz"
         },
         "tv4": {
           "version": "1.2.7",
-          "from": "tv4@>=1.1.9 <2.0.0",
+          "from": "https://registry.npmjs.org/tv4/-/tv4-1.2.7.tgz",
           "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.2.7.tgz"
         },
         "validator": {
           "version": "3.43.0",
-          "from": "validator@>=3.34.0 <4.0.0",
+          "from": "https://registry.npmjs.org/validator/-/validator-3.43.0.tgz",
           "resolved": "https://registry.npmjs.org/validator/-/validator-3.43.0.tgz"
         }
       }
@@ -209,7 +201,7 @@
           "dependencies": {
             "inherits": {
               "version": "2.0.1",
-              "from": "inherits@>=2.0.0 <3.0.0",
+              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
               "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
             },
             "statuses": {
@@ -244,14 +236,14 @@
       }
     },
     "cookie-parser": {
-      "version": "1.4.0",
-      "from": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.0.tgz",
-      "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.0.tgz",
+      "version": "1.4.1",
+      "from": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.1.tgz",
       "dependencies": {
         "cookie": {
-          "version": "0.2.2",
-          "from": "https://registry.npmjs.org/cookie/-/cookie-0.2.2.tgz",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.2.2.tgz"
+          "version": "0.2.3",
+          "from": "https://registry.npmjs.org/cookie/-/cookie-0.2.3.tgz",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.2.3.tgz"
         },
         "cookie-signature": {
           "version": "1.0.6",
@@ -295,9 +287,9 @@
           "resolved": "http://registry.npmjs.org/moment-timezone/-/moment-timezone-0.3.1.tgz",
           "dependencies": {
             "moment": {
-              "version": "2.10.6",
-              "from": "https://registry.npmjs.org/moment/-/moment-2.10.6.tgz",
-              "resolved": "https://registry.npmjs.org/moment/-/moment-2.10.6.tgz"
+              "version": "2.11.2",
+              "from": "https://registry.npmjs.org/moment/-/moment-2.11.2.tgz",
+              "resolved": "https://registry.npmjs.org/moment/-/moment-2.11.2.tgz"
             }
           }
         }
@@ -319,9 +311,9 @@
           "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
         },
         "csrf": {
-          "version": "3.0.0",
-          "from": "https://registry.npmjs.org/csrf/-/csrf-3.0.0.tgz",
-          "resolved": "https://registry.npmjs.org/csrf/-/csrf-3.0.0.tgz",
+          "version": "3.0.1",
+          "from": "https://registry.npmjs.org/csrf/-/csrf-3.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/csrf/-/csrf-3.0.1.tgz",
           "dependencies": {
             "base64-url": {
               "version": "1.2.1",
@@ -329,9 +321,9 @@
               "resolved": "https://registry.npmjs.org/base64-url/-/base64-url-1.2.1.tgz"
             },
             "rndm": {
-              "version": "1.1.1",
-              "from": "https://registry.npmjs.org/rndm/-/rndm-1.1.1.tgz",
-              "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.1.1.tgz"
+              "version": "1.2.0",
+              "from": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz",
+              "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz"
             },
             "scmp": {
               "version": "1.0.0",
@@ -339,9 +331,16 @@
               "resolved": "https://registry.npmjs.org/scmp/-/scmp-1.0.0.tgz"
             },
             "uid-safe": {
-              "version": "2.0.0",
-              "from": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.0.0.tgz",
-              "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.0.0.tgz"
+              "version": "2.1.0",
+              "from": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.0.tgz",
+              "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.0.tgz",
+              "dependencies": {
+                "random-bytes": {
+                  "version": "1.0.0",
+                  "from": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz",
+                  "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz"
+                }
+              }
             }
           }
         },
@@ -352,7 +351,7 @@
           "dependencies": {
             "inherits": {
               "version": "2.0.1",
-              "from": "inherits@>=2.0.0 <3.0.0",
+              "from": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
               "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
             },
             "statuses": {
@@ -414,9 +413,9 @@
           "resolved": "http://registry.npmjs.org/moment/-/moment-2.9.0.tgz"
         },
         "mongodb": {
-          "version": "1.4.39",
-          "from": "https://registry.npmjs.org/mongodb/-/mongodb-1.4.39.tgz",
-          "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-1.4.39.tgz",
+          "version": "1.4.40",
+          "from": "https://registry.npmjs.org/mongodb/-/mongodb-1.4.40.tgz",
+          "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-1.4.40.tgz",
           "dependencies": {
             "bson": {
               "version": "0.2.22",
@@ -443,9 +442,9 @@
               }
             },
             "readable-stream": {
-              "version": "2.0.4",
-              "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.4.tgz",
-              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.4.tgz",
+              "version": "2.0.5",
+              "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.5.tgz",
+              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.5.tgz",
               "dependencies": {
                 "core-util-is": {
                   "version": "1.0.2",
@@ -1350,14 +1349,14 @@
       }
     },
     "dockerode": {
-      "version": "2.2.7",
-      "from": "https://registry.npmjs.org/dockerode/-/dockerode-2.2.7.tgz",
-      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.2.7.tgz",
+      "version": "2.2.9",
+      "from": "https://registry.npmjs.org/dockerode/-/dockerode-2.2.9.tgz",
+      "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.2.9.tgz",
       "dependencies": {
         "docker-modem": {
-          "version": "0.2.8",
-          "from": "https://registry.npmjs.org/docker-modem/-/docker-modem-0.2.8.tgz",
-          "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-0.2.8.tgz",
+          "version": "0.3.0",
+          "from": "https://registry.npmjs.org/docker-modem/-/docker-modem-0.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-0.3.0.tgz",
           "dependencies": {
             "JSONStream": {
               "version": "0.10.0",
@@ -1386,11 +1385,6 @@
               "from": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.3.tgz",
               "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.3.tgz"
             },
-            "querystring": {
-              "version": "0.2.0",
-              "from": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
-              "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz"
-            },
             "readable-stream": {
               "version": "1.0.33",
               "from": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz",
@@ -1428,9 +1422,9 @@
       }
     },
     "ejs": {
-      "version": "2.3.4",
-      "from": "https://registry.npmjs.org/ejs/-/ejs-2.3.4.tgz",
-      "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.3.4.tgz"
+      "version": "2.4.1",
+      "from": "https://registry.npmjs.org/ejs/-/ejs-2.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.4.1.tgz"
     },
     "ejs-cli": {
       "version": "1.0.1",
@@ -1496,9 +1490,9 @@
       }
     },
     "express": {
-      "version": "4.13.3",
-      "from": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
+      "version": "4.13.4",
+      "from": "https://registry.npmjs.org/express/-/express-4.13.4.tgz",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.13.4.tgz",
       "dependencies": {
         "accepts": {
           "version": "1.2.13",
@@ -1506,14 +1500,14 @@
           "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
           "dependencies": {
             "mime-types": {
-              "version": "2.1.8",
-              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.8.tgz",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.8.tgz",
+              "version": "2.1.9",
+              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
               "dependencies": {
                 "mime-db": {
-                  "version": "1.20.0",
-                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.20.0.tgz",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.20.0.tgz"
+                  "version": "1.21.0",
+                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
                 }
               }
             },
@@ -1530,9 +1524,9 @@
           "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
         },
         "content-disposition": {
-          "version": "0.5.0",
-          "from": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz",
-          "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz"
+          "version": "0.5.1",
+          "from": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz",
+          "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz"
         },
         "content-type": {
           "version": "1.0.1",
@@ -1540,9 +1534,9 @@
           "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
         },
         "cookie": {
-          "version": "0.1.3",
-          "from": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz"
+          "version": "0.1.5",
+          "from": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz"
         },
         "cookie-signature": {
           "version": "1.0.6",
@@ -1550,14 +1544,14 @@
           "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
         },
         "depd": {
-          "version": "1.0.1",
-          "from": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz",
-          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+          "version": "1.1.0",
+          "from": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
         },
         "escape-html": {
-          "version": "1.0.2",
-          "from": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz",
-          "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz"
+          "version": "1.0.3",
+          "from": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+          "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz"
         },
         "etag": {
           "version": "1.7.0",
@@ -1565,9 +1559,9 @@
           "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
         },
         "finalhandler": {
-          "version": "0.4.0",
-          "from": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
-          "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
+          "version": "0.4.1",
+          "from": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz",
+          "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz",
           "dependencies": {
             "unpipe": {
               "version": "1.0.0",
@@ -1582,14 +1576,14 @@
           "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
         },
         "merge-descriptors": {
-          "version": "1.0.0",
-          "from": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz",
-          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
+          "version": "1.0.1",
+          "from": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz"
         },
         "methods": {
-          "version": "1.1.1",
-          "from": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz",
-          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
+          "version": "1.1.2",
+          "from": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
         },
         "on-finished": {
           "version": "2.3.0",
@@ -1604,9 +1598,9 @@
           }
         },
         "parseurl": {
-          "version": "1.3.0",
-          "from": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz",
-          "resolved": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
+          "version": "1.3.1",
+          "from": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz"
         },
         "path-to-regexp": {
           "version": "0.1.7",
@@ -1641,14 +1635,14 @@
           "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
         },
         "send": {
-          "version": "0.13.0",
-          "from": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
-          "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
+          "version": "0.13.1",
+          "from": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
+          "resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
           "dependencies": {
             "destroy": {
-              "version": "1.0.3",
-              "from": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz",
-              "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz"
+              "version": "1.0.4",
+              "from": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
+              "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
             },
             "http-errors": {
               "version": "1.3.1",
@@ -1675,29 +1669,29 @@
           }
         },
         "serve-static": {
-          "version": "1.10.0",
-          "from": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz",
-          "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
+          "version": "1.10.2",
+          "from": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.2.tgz",
+          "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.2.tgz"
         },
         "type-is": {
-          "version": "1.6.10",
-          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.10.tgz",
-          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.10.tgz",
+          "version": "1.6.11",
+          "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.11.tgz",
           "dependencies": {
             "media-typer": {
               "version": "0.3.0",
-              "from": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-              "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+              "from": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
             },
             "mime-types": {
-              "version": "2.1.8",
-              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.8.tgz",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.8.tgz",
+              "version": "2.1.9",
+              "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
               "dependencies": {
                 "mime-db": {
-                  "version": "1.20.0",
-                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.20.0.tgz",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.20.0.tgz"
+                  "version": "1.21.0",
+                  "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.21.0.tgz"
                 }
               }
             }
@@ -1716,9 +1710,9 @@
       }
     },
     "express-session": {
-      "version": "1.12.1",
-      "from": "https://registry.npmjs.org/express-session/-/express-session-1.12.1.tgz",
-      "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.12.1.tgz",
+      "version": "1.13.0",
+      "from": "https://registry.npmjs.org/express-session/-/express-session-1.13.0.tgz",
+      "resolved": "https://registry.npmjs.org/express-session/-/express-session-1.13.0.tgz",
       "dependencies": {
         "cookie": {
           "version": "0.2.3",
@@ -1731,9 +1725,9 @@
           "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
         },
         "crc": {
-          "version": "3.3.0",
-          "from": "https://registry.npmjs.org/crc/-/crc-3.3.0.tgz",
-          "resolved": "https://registry.npmjs.org/crc/-/crc-3.3.0.tgz"
+          "version": "3.4.0",
+          "from": "https://registry.npmjs.org/crc/-/crc-3.4.0.tgz",
+          "resolved": "https://registry.npmjs.org/crc/-/crc-3.4.0.tgz"
         },
         "depd": {
           "version": "1.1.0",
@@ -1746,9 +1740,9 @@
           "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz"
         },
         "parseurl": {
-          "version": "1.3.0",
-          "from": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz",
-          "resolved": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
+          "version": "1.3.1",
+          "from": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz"
         },
         "uid-safe": {
           "version": "2.0.0",
@@ -1942,14 +1936,14 @@
       }
     },
     "mysql": {
-      "version": "2.9.0",
-      "from": "https://registry.npmjs.org/mysql/-/mysql-2.9.0.tgz",
-      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.9.0.tgz",
+      "version": "2.10.2",
+      "from": "https://registry.npmjs.org/mysql/-/mysql-2.10.2.tgz",
+      "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.10.2.tgz",
       "dependencies": {
         "bignumber.js": {
-          "version": "2.0.7",
-          "from": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-2.0.7.tgz",
-          "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-2.0.7.tgz"
+          "version": "2.1.4",
+          "from": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-2.1.4.tgz",
+          "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-2.1.4.tgz"
         },
         "readable-stream": {
           "version": "1.1.13",
@@ -2013,10 +2007,15 @@
               "resolved": "http://registry.npmjs.org/buffercursor/-/buffercursor-0.0.12.tgz",
               "dependencies": {
                 "verror": {
-                  "version": "1.6.0",
-                  "from": "https://registry.npmjs.org/verror/-/verror-1.6.0.tgz",
-                  "resolved": "https://registry.npmjs.org/verror/-/verror-1.6.0.tgz",
+                  "version": "1.6.1",
+                  "from": "https://registry.npmjs.org/verror/-/verror-1.6.1.tgz",
+                  "resolved": "https://registry.npmjs.org/verror/-/verror-1.6.1.tgz",
                   "dependencies": {
+                    "core-util-is": {
+                      "version": "1.0.2",
+                      "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+                      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
+                    },
                     "extsprintf": {
                       "version": "1.2.0",
                       "from": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.2.0.tgz",
@@ -2030,15 +2029,20 @@
         }
       }
     },
+    "node-df": {
+      "version": "0.1.1",
+      "from": "https://registry.npmjs.org/node-df/-/node-df-0.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/node-df/-/node-df-0.1.1.tgz"
+    },
     "node-uuid": {
       "version": "1.4.7",
       "from": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz",
       "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz"
     },
     "nodemailer": {
-      "version": "1.10.0",
-      "from": "https://registry.npmjs.org/nodemailer/-/nodemailer-1.10.0.tgz",
-      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-1.10.0.tgz",
+      "version": "1.11.0",
+      "from": "https://registry.npmjs.org/nodemailer/-/nodemailer-1.11.0.tgz",
+      "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-1.11.0.tgz",
       "dependencies": {
         "libmime": {
           "version": "1.2.0",
@@ -2121,18 +2125,18 @@
           "resolved": "https://registry.npmjs.org/nodemailer-direct-transport/-/nodemailer-direct-transport-1.1.0.tgz",
           "dependencies": {
             "smtp-connection": {
-              "version": "1.3.1",
-              "from": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.1.tgz",
-              "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.1.tgz"
+              "version": "1.3.8",
+              "from": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.8.tgz",
+              "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.8.tgz"
             }
           }
         }
       }
     },
     "nodemailer-smtp-transport": {
-      "version": "1.0.4",
-      "from": "https://registry.npmjs.org/nodemailer-smtp-transport/-/nodemailer-smtp-transport-1.0.4.tgz",
-      "resolved": "https://registry.npmjs.org/nodemailer-smtp-transport/-/nodemailer-smtp-transport-1.0.4.tgz",
+      "version": "1.1.0",
+      "from": "https://registry.npmjs.org/nodemailer-smtp-transport/-/nodemailer-smtp-transport-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/nodemailer-smtp-transport/-/nodemailer-smtp-transport-1.1.0.tgz",
       "dependencies": {
         "clone": {
           "version": "1.0.2",
@@ -2145,16 +2149,16 @@
           "resolved": "https://registry.npmjs.org/nodemailer-wellknown/-/nodemailer-wellknown-0.1.7.tgz"
         },
         "smtp-connection": {
-          "version": "1.3.1",
-          "from": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.1.tgz",
-          "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.1.tgz"
+          "version": "1.3.8",
+          "from": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.8.tgz",
+          "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-1.3.8.tgz"
         }
       }
     },
     "oauth2orize": {
-      "version": "1.2.0",
-      "from": "https://registry.npmjs.org/oauth2orize/-/oauth2orize-1.2.0.tgz",
-      "resolved": "https://registry.npmjs.org/oauth2orize/-/oauth2orize-1.2.0.tgz",
+      "version": "1.2.2",
+      "from": "https://registry.npmjs.org/oauth2orize/-/oauth2orize-1.2.2.tgz",
+      "resolved": "https://registry.npmjs.org/oauth2orize/-/oauth2orize-1.2.2.tgz",
       "dependencies": {
         "uid2": {
           "version": "0.0.3",
@@ -2180,6 +2184,11 @@
         }
       }
     },
+    "parse-links": {
+      "version": "0.1.0",
+      "from": "parse-links@>=0.1.0 <0.2.0",
+      "resolved": "https://registry.npmjs.org/parse-links/-/parse-links-0.1.0.tgz"
+    },
     "passport": {
       "version": "0.2.2",
       "from": "https://registry.npmjs.org/passport/-/passport-0.2.2.tgz",
@@ -2287,9 +2296,9 @@
       "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.13.1.tgz"
     },
     "safetydance": {
-      "version": "0.1.0",
-      "from": "https://registry.npmjs.org/safetydance/-/safetydance-0.1.0.tgz",
-      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.1.0.tgz"
+      "version": "0.1.1",
+      "from": "https://registry.npmjs.org/safetydance/-/safetydance-0.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.1.1.tgz"
     },
     "semver": {
       "version": "4.3.6",
@@ -2317,9 +2326,9 @@
           "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
         },
         "parseurl": {
-          "version": "1.3.0",
-          "from": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz",
-          "resolved": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
+          "version": "1.3.1",
+          "from": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz"
         }
       }
     },
@@ -2336,9 +2345,9 @@
       }
     },
     "superagent": {
-      "version": "1.5.0",
-      "from": "https://registry.npmjs.org/superagent/-/superagent-1.5.0.tgz",
-      "resolved": "https://registry.npmjs.org/superagent/-/superagent-1.5.0.tgz",
+      "version": "1.7.2",
+      "from": "https://registry.npmjs.org/superagent/-/superagent-1.7.2.tgz",
+      "resolved": "https://registry.npmjs.org/superagent/-/superagent-1.7.2.tgz",
       "dependencies": {
         "qs": {
           "version": "2.3.3",
@@ -2346,24 +2355,24 @@
           "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz"
         },
         "formidable": {
-          "version": "1.0.14",
-          "from": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz",
-          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz"
+          "version": "1.0.17",
+          "from": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz",
+          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz"
         },
         "component-emitter": {
-          "version": "1.1.2",
-          "from": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz",
-          "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz"
+          "version": "1.2.0",
+          "from": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.0.tgz"
         },
         "methods": {
-          "version": "1.0.1",
-          "from": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz"
+          "version": "1.1.2",
+          "from": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
         },
         "cookiejar": {
-          "version": "2.0.1",
-          "from": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz",
-          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz"
+          "version": "2.0.6",
+          "from": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.6.tgz",
+          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.6.tgz"
         },
         "reduce-component": {
           "version": "1.0.1",
@@ -2371,9 +2380,9 @@
           "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
         },
         "extend": {
-          "version": "1.2.1",
-          "from": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz",
-          "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz"
+          "version": "3.0.0",
+          "from": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
         },
         "form-data": {
           "version": "0.2.0",
@@ -2418,7 +2427,7 @@
           "dependencies": {
             "core-util-is": {
               "version": "1.0.2",
-              "from": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
+              "from": "core-util-is@>=1.0.0 <1.1.0",
               "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
             },
             "isarray": {
@@ -2467,6 +2476,11 @@
       "from": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
       "resolved": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz"
     },
+    "tldjs": {
+      "version": "1.6.2",
+      "from": "https://registry.npmjs.org/tldjs/-/tldjs-1.6.2.tgz",
+      "resolved": "https://registry.npmjs.org/tldjs/-/tldjs-1.6.2.tgz"
+    },
     "underscore": {
       "version": "1.8.3",
       "from": "http://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz",
@@ -2483,9 +2497,9 @@
           "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz"
         },
         "nan": {
-          "version": "2.1.0",
-          "from": "https://registry.npmjs.org/nan/-/nan-2.1.0.tgz",
-          "resolved": "https://registry.npmjs.org/nan/-/nan-2.1.0.tgz"
+          "version": "2.2.0",
+          "from": "https://registry.npmjs.org/nan/-/nan-2.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/nan/-/nan-2.2.0.tgz"
         }
       }
     },
@@ -2495,9 +2509,16 @@
       "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz"
     },
     "validator": {
-      "version": "4.4.0",
-      "from": "https://registry.npmjs.org/validator/-/validator-4.4.0.tgz",
-      "resolved": "https://registry.npmjs.org/validator/-/validator-4.4.0.tgz"
+      "version": "4.8.0",
+      "from": "https://registry.npmjs.org/validator/-/validator-4.8.0.tgz",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-4.8.0.tgz",
+      "dependencies": {
+        "depd": {
+          "version": "1.1.0",
+          "from": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
+        }
+      }
     },
     "x509": {
       "version": "0.2.3",

package.json

@@ -18,7 +18,7 @@
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
     "bytes": "^2.1.0",
-    "cloudron-manifestformat": "^2.2.0",
+    "cloudron-manifestformat": "^2.3.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
@@ -42,11 +42,13 @@
     "multiparty": "^4.1.2",
     "mysql": "^2.7.0",
     "native-dns": "^0.7.0",
+    "node-df": "^0.1.1",
     "node-uuid": "^1.4.3",
     "nodemailer": "^1.3.0",
     "nodemailer-smtp-transport": "^1.0.3",
     "oauth2orize": "^1.0.1",
     "once": "^1.3.2",
+    "parse-links": "^0.1.0",
     "passport": "^0.2.2",
     "passport-http": "^0.2.2",
     "passport-http-bearer": "^1.0.1",
@@ -54,13 +56,14 @@
     "passport-oauth2-client-password": "^0.1.2",
     "password-generator": "^2.0.2",
     "proxy-middleware": "^0.13.0",
-    "safetydance": "^0.1.0",
+    "safetydance": "^0.1.1",
     "semver": "^4.3.6",
     "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
     "superagent": "^1.5.0",
     "supererror": "^0.7.1",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "tldjs": "^1.6.2",
     "underscore": "^1.7.0",
     "ursa": "^0.9.1",
     "valid-url": "^1.0.9",
@@ -70,13 +73,14 @@
   "devDependencies": {
     "apidoc": "*",
     "bootstrap-sass": "^3.3.3",
+    "deep-extend": "^0.4.1",
     "del": "^1.1.1",
     "expect.js": "*",
     "gulp": "^3.8.11",
     "gulp-autoprefixer": "^2.3.0",
     "gulp-concat": "^2.4.3",
+    "gulp-cssnano": "^2.1.0",
     "gulp-ejs": "^1.0.0",
-    "gulp-minify-css": "^1.1.3",
     "gulp-sass": "^2.0.1",
     "gulp-serve": "^1.0.0",
     "gulp-sourcemaps": "^1.5.2",

setup/INFRA_VERSION

@@ -3,12 +3,12 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=21
+INFRA_VERSION=23
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
 BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.8.0
+MYSQL_IMAGE=cloudron/mysql:0.10.0
 POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
 MONGODB_IMAGE=cloudron/mongodb:0.8.0
 REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well

setup/argparser.sh

@@ -19,6 +19,7 @@ arg_version=""
 arg_web_server_origin=""
 arg_backup_config=""
 arg_dns_config=""
+arg_update_config=""
 arg_provider=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
@@ -56,6 +57,9 @@ EOF
         arg_dns_config=$(echo "$2" | $json dnsConfig)
         [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
 
+        arg_update_config=$(echo "$2" | $json updateConfig)
+        [[ "${arg_update_config}" == "null" ]] && arg_update_config=""
+
         shift 2
         ;;
     --) break;;

setup/container/sudoers

@@ -30,3 +30,7 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
 
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
+
+Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
+

setup/container/systemd/box.service

@@ -2,6 +2,8 @@
 Description=Cloudron Admin
 OnFailure=crashnotifier@%n.service
 StopWhenUnneeded=true
+; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
+BindsTo=systemd-journald.service
 
 [Service]
 Type=idle
@@ -9,7 +11,8 @@ WorkingDirectory=/home/yellowtent/box
 Restart=always
 ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
 Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
-KillMode=process
+; kill apptask processes as well
+KillMode=control-group
 User=yellowtent
 Group=yellowtent
 MemoryLimit=200M

setup/container/systemd/cloudron.target

@@ -1,5 +1,5 @@
 [Unit]
-Description=Cloudron Smart Cloud
+Description=Cloudron Smartserver
 Documentation=https://cloudron.io/documentation.html
 StopWhenUnneeded=true
 Requires=box.service

setup/start.sh

@@ -92,13 +92,6 @@ EOF
 
 set_progress "28" "Setup collectd"
 cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
-## collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
-#mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
-## detect device, let it fail if non exists
-#[[ -b "/dev/vda1" ]] && disk_device="/dev/vda1"
-#[[ -b "/dev/xvda1" ]] && disk_device="/dev/xvda1"
-#vda1_id=$(blkid -s UUID -o value ${disk_device})
-#ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
 service collectd restart
 
 set_progress "30" "Setup nginx"
@@ -181,6 +174,14 @@ if [[ ! -z "${arg_dns_config}" ]]; then
         -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
 fi
 
+# Add Update Configuration
+if [[ ! -z "${arg_update_config}" ]]; then
+    echo "Add Update Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"update_config\", '$arg_update_config')" box
+fi
+
 # Add TLS Configuration
 if [[ ! -z "${arg_tls_config}" ]]; then
     echo "Add TLS Config"

setup/start/collectd.conf

@@ -133,10 +133,10 @@ LoadPlugin nginx
 #   Globals true
 #</LoadPlugin>
 #LoadPlugin pinba
-LoadPlugin ping
+#LoadPlugin ping
 #LoadPlugin postgresql
 #LoadPlugin powerdns
-LoadPlugin processes
+#LoadPlugin processes
 #LoadPlugin protocols
 #<LoadPlugin python>
 #   Globals true
@@ -161,7 +161,7 @@ LoadPlugin tail
 #LoadPlugin users
 #LoadPlugin uuid
 #LoadPlugin varnish
-LoadPlugin vmem
+#LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
 LoadPlugin write_graphite
@@ -193,11 +193,11 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin df>
-   FSType "tmpfs"
-   MountPoint "/dev"
+   FSType "ext4"
+   FSType "btrfs"
 
    ReportByDevice true
-   IgnoreSelected true
+   IgnoreSelected false
 
    ValuesAbsolute true
    ValuesPercentage true
@@ -212,17 +212,6 @@ LoadPlugin write_graphite
    URL "http://127.0.0.1/nginx_status"
 </Plugin>
 
-<Plugin ping>
-   Host "google.com"
-   Interval 1.0
-   Timeout 0.9
-   TTL 255
-</Plugin>
-
-<Plugin processes>
-   ProcessMatch "app" "node box.js"
-</Plugin>
-
 <Plugin swap>
    ReportByDevice false
    ReportBytes true
@@ -255,10 +244,6 @@ LoadPlugin write_graphite
    </File>
 </Plugin>
 
-<Plugin vmem>
-   Verbose false
-</Plugin>
-
 <Plugin write_graphite>
    <Node "graphing">
        Host "localhost"

setup/start/nginx/appconfig.ejs

@@ -58,12 +58,17 @@ server {
             client_max_body_size 1m;
         }
 
-        # graphite paths
-        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
-            proxy_pass   http://127.0.0.1:8000;
-            client_max_body_size 1m;
+        location ~ ^/api/v1/apps/.*/exec$ {
+            proxy_pass   http://127.0.0.1:3000;
+            proxy_read_timeout 30m;
         }
 
+        # graphite paths
+        # location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+        #     proxy_pass   http://127.0.0.1:8000;
+        #     client_max_body_size 1m;
+        # }
+
         location / {
             root   <%= sourceDir %>/webadmin/dist;
             index  index.html index.htm;

setup/start/nginx/nginx.conf

@@ -58,6 +58,14 @@ http {
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
+        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+        proxy_buffer_size       128k;
+        proxy_buffers           4 256k;
+        proxy_busy_buffers_size 256k;
+
+        # Disable check to allow unlimited body sizes
+        client_max_body_size 0;
+
         error_page 404 = @fallback;
         location @fallback {
             internal;
@@ -65,7 +73,16 @@ http {
             rewrite ^/$ /nakeddomain.html break;
         }
 
-        return 404;
+        location / {
+            internal;
+            root /home/yellowtent/box/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
+        location /api/ {
+            proxy_pass http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
     }
 
     include applications/*.conf;

setup/start/setup_infra.sh

@@ -63,8 +63,8 @@ readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
 readonly MYSQL_ROOT_HOST='${docker0_ip}'
 EOF
 mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 100m \
-    --memory-swap 200m \
+    -m 256m \
+    --memory-swap 512m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mysql:/var/lib/mysql" \
     -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \

src/addons.js

@@ -394,6 +394,7 @@ function setupSendMail(app, options, callback) {
         'MAIL_SMTP_SERVER=mail',
         'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
         'MAIL_SMTP_USERNAME=' + username,
+        'MAIL_SMTP_PASSWORD=' + hat(256), // this is ignored
         'MAIL_DOMAIN=' + config.fqdn()
     ];
 
@@ -420,7 +421,7 @@ function setupMySql(app, options, callback) {
     debugApp(app, 'Setting up mysql');
 
     var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'add', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];
 
     container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
         if (error) return callback(error);
@@ -453,7 +454,7 @@ function teardownMySql(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];
 
     debugApp(app, 'Tearing down mysql');
 
@@ -481,7 +482,7 @@ function backupMySql(app, options, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', 'backup', app.id ]);
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ]);
     cp.on('error', callback);
     cp.on('exit', function (code, signal) {
         debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
@@ -504,7 +505,7 @@ function restoreMySql(app, options, callback) {
         input.on('error', callback);
 
         // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', 'restore', app.id ]);
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ]);
         cp.on('error', callback);
         cp.on('exit', function (code, signal) {
             debugApp(app, 'restoreMySql: done %s %s', code, signal);

src/appdb.js

@@ -59,7 +59,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -92,8 +92,6 @@ function postProcess(result) {
         result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
     }
 
-    result.oauthProxy = !!result.oauthProxy;
-
     assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
     result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
     if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
@@ -179,7 +177,7 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
@@ -187,7 +185,7 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert.strictEqual(typeof memoryLimit, 'number');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
@@ -197,8 +195,8 @@ function add(id, appStoreId, manifest, location, portBindings, accessRestriction
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit) VALUES (?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -283,6 +281,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     assert.strictEqual(typeof constraints, 'string');
     assert.strictEqual(typeof callback, 'function');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
 
     var queries = [ ];
 
@@ -368,7 +367,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
         updateWithConstraints(appId, values, '', callback);
     } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
         updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_CONFIGURE || installationState === exports.ISTATE_PENDING_BACKUP) {
         updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
     } else {
         callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));

src/apphealthmonitor.js

@@ -3,6 +3,7 @@
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
+    config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apphealthmonitor'),
     docker = require('./docker.js').connection,
@@ -24,8 +25,11 @@ var gDockerEventStream = null;
 function debugApp(app) {
     assert(!app || typeof app === 'object');
 
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
+    var manifestAppId = app ? app.manifest.id : '';
+    var id = app ? app.id : '';
+
+    debug(prefix + ' ' + manifestAppId + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + id);
 }
 
 function setHealth(app, health, callback) {
@@ -89,6 +93,7 @@ function checkAppHealth(app, callback) {
         var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
         superagent
             .get(healthCheckUrl)
+            .set('Host', config.appFqdn(app.location)) // required for some apache configs with rewrite rules
             .redirects(0)
             .timeout(HEALTHCHECK_INTERVAL)
             .end(function (error, res) {
@@ -114,7 +119,7 @@ function processApps(callback) {
 
             var alive = apps
                 .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
-                .map(function (a) { return a.location; }).join(', ');
+                .map(function (a) { return (a.location || 'naked_domain') + '|' + a.manifest.id; }).join(', ');
 
             debug('apps alive: [%s]', alive);
 

src/apps.js

@@ -9,7 +9,9 @@ exports = module.exports = {
 
     get: get,
     getBySubdomain: getBySubdomain,
+    getByIpAddress: getByIpAddress,
     getAll: getAll,
+    getAllByUser: getAllByUser,
     purchase: purchase,
     install: install,
     configure: configure,
@@ -22,6 +24,7 @@ exports = module.exports = {
 
     backup: backup,
     backupApp: backupApp,
+    listBackups: listBackups,
 
     getLogs: getLogs,
 
@@ -55,8 +58,8 @@ var addons = require('./addons.js'),
     debug = require('debug')('box:apps'),
     docker = require('./docker.js'),
     fs = require('fs'),
+    groups = require('./groups.js'),
     manifestFormat = require('cloudron-manifestformat'),
-    once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -73,8 +76,6 @@ var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
     RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
 
@@ -194,9 +195,38 @@ function validateAccessRestriction(accessRestriction) {
 
     if (accessRestriction === null) return null;
 
-    if (!accessRestriction.users || !Array.isArray(accessRestriction.users)) return new Error('users array property required');
-    if (accessRestriction.users.length === 0) return new Error('users array cannot be empty');
-    if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+    var noUsers = true, noGroups = true;
+
+    if (accessRestriction.users) {
+        if (!Array.isArray(accessRestriction.users)) return new Error('users array property required');
+        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+        noUsers = accessRestriction.users.length === 0;
+    }
+
+    if (accessRestriction.groups) {
+        if (!Array.isArray(accessRestriction.groups)) return new Error('groups array property required');
+        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new Error('All groups have to be strings');
+        noGroups = accessRestriction.groups.length === 0;
+    }
+
+    if (noUsers && noGroups) return new Error('users and groups array cannot both be empty');
+
+    return null;
+}
+
+function validateMemoryLimit(manifest, memoryLimit) {
+    assert.strictEqual(typeof manifest, 'object');
+    assert.strictEqual(typeof memoryLimit, 'number');
+
+    var min = manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+    var max = (4096 * 1024 * 1024);
+
+    // allow 0, which indicates that it is not set, the one from the manifest will be choosen but we don't commit any user value
+    // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
+    if (memoryLimit === 0) return null;
+
+    if (memoryLimit < min) return new Error('memoryLimit too small');
+    if (memoryLimit > max) return new Error('memoryLimit too large');
 
     return null;
 }
@@ -228,12 +258,26 @@ function getIconUrlSync(app) {
     return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
 }
 
-function hasAccessTo(app, user) {
+function hasAccessTo(app, user, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (app.accessRestriction === null) return true;
-    return app.accessRestriction.users.some(function (e) { return e === user.id; });
+    if (app.accessRestriction === null) return callback(null, true);
+
+    // check user access
+    if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return callback(null, true);
+
+    // check group access
+    if (!app.accessRestriction.groups) return callback(null, false);
+
+    async.some(app.accessRestriction.groups, function (groupId, iteratorDone) {
+        groups.isMember(groupId, user.id, function (error, member) {
+            iteratorDone(!error && member); // async.some does not take error argument in callback
+        });
+    }, function (result) {
+        callback(null, result);
+    });
 }
 
 function get(appId, callback) {
@@ -266,6 +310,25 @@ function getBySubdomain(subdomain, callback) {
     });
 }
 
+function getByIpAddress(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.getContainerIdByIp(ip, function (error, containerId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        appdb.getByContainerId(containerId, function (error, app) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            app.iconUrl = getIconUrlSync(app);
+            app.fqdn = config.appFqdn(app.location);
+
+            callback(null, app);
+        });
+    });
+}
+
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -281,6 +344,21 @@ function getAll(callback) {
     });
 }
 
+function getAllByUser(user, callback) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, result) {
+        if (error) return callback(error);
+
+        async.filter(result, function (app, callback) {
+            hasAccessTo(app, user, function (error, hasAccess) {
+                callback(hasAccess);
+            });
+        }, callback.bind(null, null));  // never error
+    });
+}
+
 function purchase(appStoreId, callback) {
     assert.strictEqual(typeof appStoreId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -303,17 +381,17 @@ function purchase(appStoreId, callback) {
     });
 }
 
-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, icon, cert, key, callback) {
+function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
     assert(!icon || typeof icon === 'string');
     assert(cert === null || typeof cert === 'string');
     assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof memoryLimit, 'number');
     assert.strictEqual(typeof callback, 'function');
 
     var error = manifestFormat.parse(manifest);
@@ -331,6 +409,12 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     error = validateAccessRestriction(accessRestriction);
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+    error = validateMemoryLimit(manifest, memoryLimit);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    // memoryLimit might come in as 0 if not specified
+    memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
     // singleUser mode requires accessRestriction to contain exactly one user
     if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
     if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
@@ -351,7 +435,7 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     purchase(appStoreId, function (error) {
         if (error) return callback(error);
 
-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, oauthProxy, function (error) {
+        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, function (error) {
             if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -368,14 +452,14 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     });
 }
 
-function configure(appId, location, portBindings, accessRestriction, oauthProxy, cert, key, callback) {
+function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
     assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
     assert(cert === null || typeof cert === 'string');
     assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof memoryLimit, 'number');
     assert.strictEqual(typeof callback, 'function');
 
     var error = validateHostname(location, config.fqdn());
@@ -394,6 +478,12 @@ function configure(appId, location, portBindings, accessRestriction, oauthProxy,
         error = validatePortBindings(portBindings, app.manifest.tcpPorts);
         if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+        error = validateMemoryLimit(app.manifest, memoryLimit);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+        // memoryLimit might come in as 0 if not specified
+        memoryLimit = memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
         // save cert to data/box/certs
         if (cert && key) {
             if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
@@ -403,14 +493,14 @@ function configure(appId, location, portBindings, accessRestriction, oauthProxy,
         var values = {
             location: location.toLowerCase(),
             accessRestriction: accessRestriction,
-            oauthProxy: oauthProxy,
             portBindings: portBindings,
+            memoryLimit: memoryLimit,
 
             oldConfig: {
                 location: app.location,
                 accessRestriction: app.accessRestriction,
                 portBindings: app.portBindings,
-                oauthProxy: app.oauthProxy
+                memoryLimit: app.memoryLimit
             }
         };
 
@@ -459,14 +549,30 @@ function update(appId, force, manifest, portBindings, icon, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
+        var appStoreId = app.appStoreId;
+
+        // prevent user from installing a app with different manifest id over an existing app
+        // this allows cloudron install -f --app <appid> for an app installed from the appStore
+        if (app.manifest.id !== manifest.id) {
+            if (!force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
+            // clear appStoreId so that this app does not get updates anymore. this will mark is a dev app
+            appStoreId = '';
+        }
+
+        // Ensure we update the memory limit in case the new app requires more memory as a minimum
+        var memoryLimit = manifest.memoryLimit ? (app.memoryLimit < manifest.memoryLimit ? manifest.memoryLimit : app.memoryLimit) : app.memoryLimit;
+
         var values = {
+            appStoreId: appStoreId,
             manifest: manifest,
             portBindings: portBindings,
+            memoryLimit: memoryLimit,
+
             oldConfig: {
                 manifest: app.manifest,
                 portBindings: app.portBindings,
                 accessRestriction: app.accessRestriction,
-                oauthProxy: app.oauthProxy
+                memoryLimit: app.memoryLimit
             }
         };
 
@@ -552,12 +658,13 @@ function restore(appId, callback) {
             values = {
                 manifest: restoreConfig.manifest,
                 portBindings: restoreConfig.portBindings,
+                memoryLimit: restoreConfig.memoryLimit,
 
                 oldConfig: {
                     location: app.location,
                     accessRestriction: app.accessRestriction,
-                    oauthProxy: app.oauthProxy,
                     portBindings: app.portBindings,
+                    memoryLimit: app.memoryLimit,
                     manifest: app.manifest
                 }
             };
@@ -580,13 +687,13 @@ function uninstall(appId, callback) {
 
     debug('Will uninstall app with id:%s', appId);
 
-    appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UNINSTALL, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+    taskmanager.stopAppTask(appId, function () {
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UNINSTALL, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        taskmanager.restartAppTask(appId); // since uninstall is allowed from any state, kill current task
-
-        callback(null);
+            taskmanager.startAppTask(appId, callback);
+        });
     });
 }
 
@@ -648,13 +755,17 @@ function exec(appId, options, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
+        if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
+            return callback(new AppsError(AppsError.BAD_STATE, 'App not installed or running'));
+        }
+
         var container = docker.connection.getContainer(app.containerId);
 
        var execOptions = {
             AttachStdin: true,
             AttachStdout: true,
             AttachStderr: true,
-            Tty: true,
+            Tty: options.tty,
             Cmd: cmd
         };
 
@@ -662,7 +773,7 @@ function exec(appId, options, callback) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
             var startOptions = {
                 Detach: false,
-                Tty: true,
+                Tty: options.tty,
                 stdin: true // this is a dockerode option that enabled openStdin in the modem
             };
             exec.start(startOptions, function(error, stream) {
@@ -765,16 +876,16 @@ function createNewBackup(app, addonsToBackup, callback) {
     assert(!addonsToBackup || typeof addonsToBackup, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    backups.getBackupUrl(app, function (error, result) {
+    backups.getAppBackupUrl(app, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id);
+        debugApp(app, 'backupApp: backup url:%s backup config url:%s', result.url, result.configUrl);
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
             addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.configUrl, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -805,7 +916,7 @@ function backupApp(app, addonsToBackup, callback) {
             location: app.location,
             portBindings: app.portBindings,
             accessRestriction: app.accessRestriction,
-            oauthProxy: app.oauthProxy
+            memoryLimit: app.memoryLimit
         };
         backupFunction = createNewBackup.bind(null, app, addonsToBackup);
 
@@ -831,9 +942,9 @@ function backup(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    get(appId, function (error, app) {
-        if (error && error.reason === AppsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+    appdb.exists(appId, function (error, exists) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if (!exists) return callback(new AppsError(AppsError.NOT_FOUND));
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_BACKUP, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -846,13 +957,14 @@ function backup(appId, callback) {
     });
 }
 
-function restoreApp(app, addonsToRestore, callback) {
+function restoreApp(app, addonsToRestore, backupId, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
     assert(app.lastBackupId);
 
-    backups.getRestoreUrl(app.lastBackupId, function (error, result) {
+    backups.getRestoreUrl(backupId, function (error, result) {
         if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -865,3 +977,22 @@ function restoreApp(app, addonsToRestore, callback) {
         });
     });
 }
+
+function listBackups(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.exists(appId, function (error, exists) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if (!exists) return callback(new AppsError(AppsError.NOT_FOUND));
+
+        backups.getByAppIdPaged(page, perPage, appId, function (error, results) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            callback(null, results);
+        });
+    });
+}

src/apptask.js

@@ -162,7 +162,7 @@ function allocateOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!app.oauthProxy) return callback(null);
+    if (!nginx.requiresOAuthProxy(app)) return callback(null);
 
     var id = 'cid-' + uuid.v4();
     var clientSecret = hat(256);
@@ -464,6 +464,8 @@ function restore(app, callback) {
         return install(app, callback);
     }
 
+    var backupId = app.lastBackupId;
+
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         unconfigureNginx.bind(null, app),
@@ -499,7 +501,7 @@ function restore(app, callback) {
         createVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons),
+        apps.restoreApp.bind(null, app, app.manifest.addons, backupId),
 
         updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
         createContainer.bind(null, app),
@@ -596,15 +598,21 @@ function update(app, callback) {
     debugApp(app, 'Updating to %s', safe.query(app, 'manifest.version'));
 
     // app does not want these addons anymore
+    // FIXME: this does not handle option changes (like multipleDatabases)
     var unusedAddons = _.omit(app.oldConfig.manifest.addons, Object.keys(app.manifest.addons));
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '0, Verify manifest' }),
         verifyManifest.bind(null, app),
 
+        // download new image before app is stopped. this is so we can reduce downtime
+        // and also not remove the 'common' layers when the old image is deleted
+        updateApp.bind(null, app, { installationProgress: '15, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
         // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
         // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
-        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        updateApp.bind(null, app, { installationProgress: '25, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainers.bind(null, app),
@@ -620,17 +628,14 @@ function update(app, callback) {
             if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);
 
             async.series([
-                updateApp.bind(null, app, { installationProgress: '20, Backup app' }),
+                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
                 apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
             ], next);
         },
 
-        updateApp.bind(null, app, { installationProgress: '35, Downloading icon' }),
+        updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '45, Downloading image' }),
-        docker.downloadImage.bind(null, app.manifest),
-
         updateApp.bind(null, app, { installationProgress: '70, Updating addons' }),
         addons.setupAddons.bind(null, app, app.manifest.addons),
 
@@ -695,7 +700,13 @@ function uninstall(app, callback) {
 
         updateApp.bind(null, app, { installationProgress: '95, Remove app from database' }),
         appdb.del.bind(null, app.id)
-    ], callback);
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error uninstalling app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
 }
 
 function runApp(app, callback) {
@@ -758,7 +769,7 @@ function startTask(appId, callback) {
         case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
         case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
         case appdb.ISTATE_ERROR:
-            debugApp(app, 'Apptask launched with error states.');
+            debugApp(app, 'Internal error. apptask launched with error status.');
             return callback(null);
         default:
             debugApp(app, 'apptask launched with invalid command');

src/auth.js

@@ -16,6 +16,7 @@ var assert = require('assert'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
     crypto = require('crypto'),
+    groups = require('./groups'),
     passport = require('passport'),
     tokendb = require('./tokendb'),
     user = require('./user'),
@@ -123,7 +124,14 @@ function initialize(callback) {
                 // amend the tokenType of the token owner
                 user.tokenType = tokenType;
 
-                callback(null, user, info);
+                // amend the admin flag
+                groups.isMember(groups.ADMIN_GROUP_ID, user.id, function (error, isAdmin) {
+                    if (error) return callback(error);
+
+                    user.admin = isAdmin;
+
+                    callback(null, user, info);
+                });
             });
         });
     }));

src/backupdb.js

@@ -0,0 +1,116 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var BACKUPS_FIELDS = [ 'filename', 'creationTime', 'version', 'type', 'dependsOn', 'state' ];
+
+exports = module.exports = {
+    add: add,
+    getPaged: getPaged,
+    get: get,
+    del: del,
+    getByAppIdPaged: getByAppIdPaged,
+
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+}
+
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND filename LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, 'appbackup\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function get(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE filename = ? AND type = ? AND state = ? ORDER BY creationTime DESC',
+        [ filename, exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function add(backup, callback) {
+    assert(backup && typeof backup === 'object');
+    assert.strictEqual(typeof backup.filename, 'string');
+    assert.strictEqual(typeof backup.version, 'string');
+    assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
+    assert(util.isArray(backup.dependsOn));
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = backup.creationTime || new Date(); // allow tests to set the time
+
+    database.query('INSERT INTO backups (filename, version, type, creationTime, state, dependsOn) VALUES (?, ?, ?, ?, ?, ?)',
+        [ backup.filename, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(',') ],
+        function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE filename=?', [ filename ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}

src/backups.js

@@ -3,15 +3,18 @@
 exports = module.exports = {
     BackupsError: BackupsError,
 
-    getAllPaged: getAllPaged,
+    getPaged: getPaged,
+    getByAppIdPaged: getByAppIdPaged,
 
     getBackupUrl: getBackupUrl,
+    getAppBackupUrl: getAppBackupUrl,
     getRestoreUrl: getRestoreUrl,
 
     copyLastBackup: copyLastBackup
 };
 
 var assert = require('assert'),
+    backupdb = require('./backupdb.js'),
     caas = require('./storage/caas.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
@@ -51,32 +54,38 @@ function api(provider) {
     }
 }
 
-function getAllPaged(page, perPage, callback) {
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
     assert.strictEqual(typeof callback, 'function');
 
-    settings.getBackupConfig(function (error, backupConfig) {
+    backupdb.getPaged(page, perPage, function (error, results) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
-            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-
-            return callback(null, backups); // [ { creationTime, restoreKey } ] sorted by time (latest first
-        });
+        callback(null, results);
     });
 }
 
-function getBackupUrl(app, callback) {
-    assert(!app || typeof app === 'object');
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var filename = '';
-    if (app) {
-        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    } else {
-        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
-    }
+    backupdb.getByAppIdPaged(page, perPage, appId, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getBackupUrl(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
+    var filename = filebase + '.tar.gz';
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
@@ -93,7 +102,48 @@ function getBackupUrl(app, callback) {
 
             debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
 
-            callback(null, obj);
+            backupdb.add({ filename: filename, creationTime: now, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, obj);
+            });
+        });
+    });
+}
+
+function getAppBackupUrl(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
+    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
+ 
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getSignedUploadUrl(backupConfig, configFilename, function (error, configResult) {
+            if (error) return callback(error);
+
+            api(backupConfig.provider).getSignedUploadUrl(backupConfig, dataFilename, function (error, dataResult) {
+                if (error) return callback(error);
+
+                var obj = {
+                    id: dataFilename,
+                    url: dataResult.url,
+                    configUrl: configResult.url,
+                    sessionToken: dataResult.sessionToken, // this token can be used for both config and data upload
+                    backupKey: backupConfig.key // only data is encrypted
+                };
+
+                debug('getAppBackupUrl: %j', obj);
+
+                backupdb.add({ filename: dataFilename, creationTime: now, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                    if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                    callback(null, obj);
+                });
+            });
         });
     });
 }
@@ -124,19 +174,27 @@ function getRestoreUrl(backupId, callback) {
 }
 
 function copyLastBackup(app, callback) {
-    assert(app && typeof app === 'object');
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof app.lastBackupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, (new Date()).toISOString(), app.manifest.version);
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilename, function (error) {
+        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
             if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
-            return callback(null, toFilename);
+            // TODO change that logic by adjusting app.lastBackupId to not contain the file type
+            var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';
+
+            api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+                return callback(null, toFilenameArchive);
+            });
         });
     });
 }

src/cert/acme.js

@@ -1,5 +1,3 @@
-/* jslint node:true */
-
 'use strict';
 
 var assert = require('assert'),
@@ -7,6 +5,7 @@ var assert = require('assert'),
     crypto = require('crypto'),
     debug = require('debug')('box:cert/acme'),
     fs = require('fs'),
+    parseLinks = require('parse-links'),
     path = require('path'),
     paths = require('../paths.js'),
     safe = require('safetydance'),
@@ -58,7 +57,6 @@ function Acme(options) {
     this.caOrigin = options.prod ? CA_PROD : CA_STAGING;
     this.accountKeyPem = null; // Buffer
     this.email = options.email;
-    this.chainPem = options.prod ? safe.fs.readFileSync(__dirname + '/lets-encrypt-x1-cross-signed.pem.txt') : new Buffer('');
 }
 
 Acme.prototype.getNonce = function (callback) {
@@ -304,7 +302,7 @@ Acme.prototype.signCertificate = function (domain, csrDer, callback) {
 
         if (!certUrl) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Missing location in downloadCertificate'));
 
-        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // for renewal
+        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // maybe use for renewal
 
         return callback(null, result.headers.location);
     });
@@ -315,25 +313,57 @@ Acme.prototype.createKeyAndCsr = function (domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var outdir = paths.APP_CERTS_DIR;
+    var csrFile = path.join(outdir, domain + '.csr');
+    var privateKeyFile = path.join(outdir, domain + '.key');
     var execSync = safe.child_process.execSync;
 
-    var privateKeyFile = path.join(outdir, domain + '.key');
-    var key = execSync('openssl genrsa 4096');
-    if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (safe.fs.existsSync(privateKeyFile)) {
+        // in some old releases, csr file was corrupt. so always regenerate it
+        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
+    } else {
+        var key = execSync('openssl genrsa 4096');
+        if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
 
-    debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+    }
 
     var csrDer = execSync(util.format('openssl req -new -key %s -outform DER -subj /CN=%s', privateKeyFile, domain));
     if (!csrDer) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    var csrFile = path.join(outdir, domain + '.csr');
-    if (!safe.fs.writeFileSync(csrFile, csrFile)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error)); // bookkeeping
 
     debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
 
     callback(null, csrDer);
 };
 
+// TODO: download the chain in a loop following 'up' header
+Acme.prototype.downloadChain = function (linkHeader, callback) {
+    if (!linkHeader) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Empty link header when downloading certificate chain');
+
+    var linkInfo = parseLinks(linkHeader);
+    if (!linkInfo || !linkInfo.up) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Failed to parse link header when downloading certificate chain'); 
+
+    debug('downloadChain: downloading from %s', this.caOrigin + linkInfo.up);
+
+    superagent.get(this.caOrigin + linkInfo.up).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var chainDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        var chainPem = execSync('openssl x509 -inform DER -outform PEM', { input: chainDer }); // this is really just base64 encoding with header
+        if (!chainPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        callback(null, chainPem);
+    });
+};
+
 Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof certUrl, 'string');
@@ -355,18 +385,22 @@ Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
         var execSync = safe.child_process.execSync;
 
         safe.fs.writeFileSync(path.join(outdir, domain + '.der'), certificateDer);
-        debug('downloadCertificate: cert der file saved');
+        debug('downloadCertificate: cert der file for %s saved', domain);
 
         var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { input: certificateDer }); // this is really just base64 encoding with header
         if (!certificatePem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
 
-        var certificateFile = path.join(outdir, domain + '.cert');
-        var fullChainPem = Buffer.concat([certificatePem, that.chainPem]);
-        if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        that.downloadChain(result.header['link'], function (error, chainPem) {
+            if (error) return callback(error);
 
-        debug('downloadCertificate: cert file saved at %s', certificateFile);
+            var certificateFile = path.join(outdir, domain + '.cert');
+            var fullChainPem = Buffer.concat([certificatePem, chainPem]);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
 
-        callback();
+            debug('downloadCertificate: cert file for %s saved at %s', domain, certificateFile);
+
+            callback();
+        });
     });
 };
 
@@ -414,21 +448,11 @@ Acme.prototype.getCertificate = function (domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var outdir = paths.APP_CERTS_DIR;
-    var certUrl = safe.fs.readFileSync(path.join(outdir, domain + '.url'), 'utf8');
-    var certificateGetter;
-
-    if (certUrl) {
-        debug('getCertificate: renewing existing cert for %s from %s', domain, certUrl);
-        certificateGetter = this.downloadCertificate.bind(this, domain, certUrl);
-    } else {
-        debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
-        certificateGetter = this.acmeFlow.bind(this, domain);
-    }
-
-    certificateGetter(function (error) {
+    debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
+    this.acmeFlow(domain, function (error) {
         if (error) return callback(error);
 
+        var outdir = paths.APP_CERTS_DIR;
         callback(null, path.join(outdir, domain + '.cert'), path.join(outdir, domain + '.key'));
     });
 };

src/cert/lets-encrypt-x1-cross-signed.pem.txt

@@ -1,27 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEqDCCA5CgAwIBAgIRAJgT9HUT5XULQ+dDHpceRL0wDQYJKoZIhvcNAQELBQAw
-PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD
-Ew5EU1QgUm9vdCBDQSBYMzAeFw0xNTEwMTkyMjMzMzZaFw0yMDEwMTkyMjMzMzZa
-MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
-ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMTCCASIwDQYJKoZIhvcNAQEBBQAD
-ggEPADCCAQoCggEBAJzTDPBa5S5Ht3JdN4OzaGMw6tc1Jhkl4b2+NfFwki+3uEtB
-BaupnjUIWOyxKsRohwuj43Xk5vOnYnG6eYFgH9eRmp/z0HhncchpDpWRz/7mmelg
-PEjMfspNdxIknUcbWuu57B43ABycrHunBerOSuu9QeU2mLnL/W08lmjfIypCkAyG
-dGfIf6WauFJhFBM/ZemCh8vb+g5W9oaJ84U/l4avsNwa72sNlRZ9xCugZbKZBDZ1
-gGusSvMbkEl4L6KWTyogJSkExnTA0DHNjzE4lRa6qDO4Q/GxH8Mwf6J5MRM9LTb4
-4/zyM2q5OTHFr8SNDR1kFjOq+oQpttQLwNh9w5MCAwEAAaOCAZIwggGOMBIGA1Ud
-EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMH8GCCsGAQUFBwEBBHMwcTAy
-BggrBgEFBQcwAYYmaHR0cDovL2lzcmcudHJ1c3RpZC5vY3NwLmlkZW50cnVzdC5j
-b20wOwYIKwYBBQUHMAKGL2h0dHA6Ly9hcHBzLmlkZW50cnVzdC5jb20vcm9vdHMv
-ZHN0cm9vdGNheDMucDdjMB8GA1UdIwQYMBaAFMSnsaR7LHH62+FLkHX/xBVghYkQ
-MFQGA1UdIARNMEswCAYGZ4EMAQIBMD8GCysGAQQBgt8TAQEBMDAwLgYIKwYBBQUH
-AgEWImh0dHA6Ly9jcHMucm9vdC14MS5sZXRzZW5jcnlwdC5vcmcwPAYDVR0fBDUw
-MzAxoC+gLYYraHR0cDovL2NybC5pZGVudHJ1c3QuY29tL0RTVFJPT1RDQVgzQ1JM
-LmNybDATBgNVHR4EDDAKoQgwBoIELm1pbDAdBgNVHQ4EFgQUqEpqYwR93brm0Tm3
-pkVl7/Oo7KEwDQYJKoZIhvcNAQELBQADggEBANHIIkus7+MJiZZQsY14cCoBG1hd
-v0J20/FyWo5ppnfjL78S2k4s2GLRJ7iD9ZDKErndvbNFGcsW+9kKK/TnY21hp4Dd
-ITv8S9ZYQ7oaoqs7HwhEMY9sibED4aXw09xrJZTC9zK1uIfW6t5dHQjuOWv+HHoW
-ZnupyxpsEUlEaFb+/SCI4KCSBdAsYxAcsHYI5xxEI4LutHp6s3OT2FuO90WfdsIk
-6q78OMSdn875bNjdBYAqxUp2/LEIHfDBkLoQz0hFJmwAbYahqKaLn73PAAm1X2kj
-f1w8DdnkabOLGeOVcj9LQ+s67vBykx4anTjURkbqZslUEUsn2k5xeua2zUk=
------END CERTIFICATE-----

src/certificates.js

@@ -1,8 +1,7 @@
-/* jslint node:true */
-
 'use strict';
 
 var acme = require('./cert/acme.js'),
+    apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     caas = require('./cert/caas.js'),
@@ -11,12 +10,14 @@ var acme = require('./cert/acme.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:src/certificates'),
     fs = require('fs'),
+    mailer = require('./mailer.js'),
     nginx = require('./nginx.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
+    tld = require('tldjs'),
     user = require('./user.js'),
     util = require('util'),
     waitForDns = require('./waitfordns.js'),
@@ -88,7 +89,8 @@ function installAdminCertificate(callback) {
         sysinfo.getIp(function (error, ip) {
             if (error) return callback(error);
 
-            waitForDns(config.adminFqdn(), ip, config.fqdn(), function (error) {
+            var zoneName = tld.getDomain(config.fqdn());
+            waitForDns(config.adminFqdn(), ip, zoneName, function (error) {
                 if (error) return callback(error); // this cannot happen because we retry forever
 
                 ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
@@ -104,41 +106,84 @@ function installAdminCertificate(callback) {
     });
 }
 
-function needsRenewalSync(certFilePath) {
-    var result = safe.child_process.execSync('openssl x509 -checkend %s -in %s', 60 * 60 * 24 * 5, certFilePath);
+function isExpiringSync(certFilePath, hours) {
+    assert.strictEqual(typeof certFilePath, 'string');
+    assert.strictEqual(typeof hours, 'number');
 
-    return result === null; // command errored
+    if (!fs.existsSync(certFilePath)) return 2; // not found
+
+    var result = safe.child_process.spawnSync('/usr/bin/openssl', [ 'x509', '-checkend', String(60 * 60 * hours), '-in', certFilePath ]);
+
+    debug('isExpiringSync: %s %s %s', certFilePath, result.stdout.toString('utf8').trim(), result.status);
+
+    return result.status === 1; // 1 - expired 0 - not expired
 }
 
 function autoRenew(callback) {
     debug('autoRenew: Checking certificates for renewal');
     callback = callback || NOOP_CALLBACK;
 
-    var filenames = safe.fs.readdirSync(paths.APP_CERTS_DIR);
-    if (!filenames) {
-        debug('autoRenew: Error getting filenames: %s', safe.error.message);
-        return;
-    }
-
-    var certs = filenames.filter(function (f) {
-        return f.match(/\.cert$/) !== null && needsRenewalSync(path.join(paths.APP_CERTS_DIR, f));
-    });
-
-    debug('autoRenew: %j needs to be renewed', certs);
-
-    getApi(function (error, api, apiOptions) {
+    apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        async.eachSeries(certs, function iterator(cert, iteratorCallback) {
-            var domain = cert.match(/^(.*)\.cert$/)[1];
-            if (domain === 'host') return iteratorCallback(); // cannot renew fallback cert
+        allApps.push({ location: 'my' }); // inject fake webadmin app
 
-            debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+        var expiringApps = [ ];
+        for (var i = 0; i < allApps.length; i++) {
+            var appDomain = config.appFqdn(allApps[i].location);
+            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
+            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
-            api.getCertificate(domain, apiOptions, function (error) {
-                if (error) debug('autoRenew: could not renew cert for %s', domain, error);
+            if (!safe.fs.existsSync(keyFilePath)) {
+                debug('autoRenew: no existing key file for %s. skipping', appDomain);
+                continue;
+            }
 
-                iteratorCallback(); // move on to next cert
+            if (isExpiringSync(certFilePath, 24 * 30)) { // expired or not found
+                expiringApps.push(allApps[i]);
+            }
+        }
+
+        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return config.appFqdn(a.location); }));
+
+        getApi(function (error, api, apiOptions) {
+            if (error) return callback(error);
+
+            async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+                var domain = config.appFqdn(app.location);
+                debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+
+                api.getCertificate(domain, apiOptions, function (error) {
+                    var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+                    var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+                    mailer.certificateRenewed(domain, error ? error.message : '');
+
+                    if (error) {
+                        debug('autoRenew: could not renew cert for %s because %s', domain, error);
+
+                        // check if we should fallback if we expire in the coming day
+                        if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
+
+                        debug('autoRenew: using fallback certs for %s since it expires soon', domain, error);
+
+                        certFilePath = 'cert/host.cert';
+                        keyFilePath = 'cert/host.key';
+                    } else {
+                        debug('autoRenew: certificate for %s renewed', domain);
+                    }
+
+                    // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
+                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                        nginx.configureAdmin.bind(null, certFilePath, keyFilePath)
+                        : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
+
+                    configureFunc(function (ignoredError) {
+                        if (ignoredError) debug('fallbackExpiredCertificates: error reconfiguring app', ignoredError);
+
+                        iteratorCallback(); // move to next app
+                    });
+                });
             });
         });
     });
@@ -237,11 +282,11 @@ function ensureCertificate(domain, callback) {
     if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
         debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);
 
-        if (!needsRenewalSync(userCertFilePath)) return callback(null, userCertFilePath, userKeyFilePath);
-
-        debug('ensureCertificate: %s cert require renewal', domain);
+        if (!isExpiringSync(userCertFilePath, 24 * 1)) return callback(null, userCertFilePath, userKeyFilePath);
     }
 
+    debug('ensureCertificate: %s cert require renewal', domain);
+
     getApi(function (error, api, apiOptions) {
         if (error) return callback(error);
 

src/cloudron.js

@@ -16,12 +16,14 @@ exports = module.exports = {
     updateToLatest: updateToLatest,
     update: update,
     reboot: reboot,
-    migrate: migrate,
     backup: backup,
+    retire: retire,
     ensureBackup: ensureBackup,
 
     isConfiguredSync: isConfiguredSync,
 
+    checkDiskSpace: checkDiskSpace,
+
     events: new (require('events').EventEmitter)(),
 
     EVENT_ACTIVATED: 'activated',
@@ -37,8 +39,10 @@ var apps = require('./apps.js'),
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
     debug = require('debug')('box:cloudron'),
+    df = require('node-df'),
     fs = require('fs'),
     locker = require('./locker.js'),
+    mailer = require('./mailer.js'),
     os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -60,7 +64,8 @@ var apps = require('./apps.js'),
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
     BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
-    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
+    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
+    RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');
 
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
 
@@ -196,10 +201,11 @@ function setTimeZone(ip, callback) {
     });
 }
 
-function activate(username, password, email, ip, callback) {
+function activate(username, password, email, displayName, ip, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof displayName, 'string');
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
@@ -207,7 +213,7 @@ function activate(username, password, email, ip, callback) {
 
     setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
-    user.createOwner(username, password, email, function (error, userObject) {
+    user.createOwner(username, password, email, displayName, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
         if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
         if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
@@ -366,16 +372,16 @@ function txtRecordsWithSpf(callback) {
         for (i = 0; i < txtRecords.length; i++) {
             if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF
 
-            validSpf = txtRecords[i].indexOf(' a:' + config.fqdn() + ' ') !== -1;
+            validSpf = txtRecords[i].indexOf(' a:' + config.adminFqdn() + ' ') !== -1;
             break;
         }
 
         if (validSpf) return callback(null, null);
 
         if (i == txtRecords.length) {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ~all"';
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ~all"';
         } else {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
         }
 
         return callback(null, txtRecords);
@@ -402,7 +408,6 @@ function addDnsRecords() {
     sysinfo.getIp(function (error, ip) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
         var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
@@ -414,6 +419,9 @@ function addDnsRecords() {
             records.push(webadminRecord);
             records.push(dkimRecord);
         } else {
+            // for custom domains, we show a nakeddomain.html page
+            var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
+
             records.push(nakedDomainRecord);
             records.push(webadminRecord);
             records.push(dkimRecord);
@@ -453,49 +461,6 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function migrate(size, region, callback) {
-    assert.strictEqual(typeof size, 'string');
-    assert.strictEqual(typeof region, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_MIGRATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    function unlock(error) {
-        if (error) {
-            debug('Failed to migrate', error);
-            locker.unlock(locker.OP_MIGRATE);
-        } else {
-            debug('Migration initiated successfully');
-            // do not unlock; cloudron is migrating
-        }
-
-        return;
-    }
-
-    // initiate the migration in the background
-    backupBoxAndApps(function (error, restoreKey) {
-        if (error) return unlock(error);
-
-        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
-
-        superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send({ size: size, region: region, restoreKey: restoreKey })
-          .end(function (error, result) {
-            if (error && !error.response) return unlock(error);
-            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
-
-            return unlock(null);
-        });
-    });
-
-    callback(null);
-}
-
 function update(boxUpdateInfo, callback) {
     assert.strictEqual(typeof boxUpdateInfo, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -509,7 +474,12 @@ function update(boxUpdateInfo, callback) {
     progress.set(progress.UPDATE, 0, 'Starting');
 
     // initiate the update/upgrade but do not wait for it
-    if (boxUpdateInfo.upgrade) {
+    if (config.version().match(/[-+]/) !== null && config.version().replace(/[-+].*/, '') === boxUpdateInfo.version) {
+        doShortCircuitUpdate(boxUpdateInfo, function (error) {
+            if (error) debug('Short-circuit update failed', error);
+            locker.unlock(locker.OP_BOX_UPDATE);
+        });
+    } else if (boxUpdateInfo.upgrade) {
         debug('Starting upgrade');
         doUpgrade(boxUpdateInfo, function (error) {
             if (error) {
@@ -540,6 +510,16 @@ function updateToLatest(callback) {
     update(boxUpdateInfo, callback);
 }
 
+function doShortCircuitUpdate(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
+
+    debug('Starting short-circuit from prerelease version %s to release version %s', config.version(), boxUpdateInfo.version);
+    config.setVersion(boxUpdateInfo.version);
+    progress.clear(progress.UPDATE);
+    updateChecker.resetUpdateInfo();
+    callback();
+}
+
 function doUpgrade(boxUpdateInfo, callback) {
     assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
 
@@ -563,8 +543,8 @@ function doUpgrade(boxUpdateInfo, callback) {
             progress.set(progress.UPDATE, 10, 'Updating base system');
 
             // no need to unlock since this is the last thing we ever do on this box
-
-            callback(null);
+            callback();
+            retire();
         });
     });
 }
@@ -646,9 +626,9 @@ function backup(callback) {
 }
 
 function ensureBackup(callback) {
-    callback = callback || function () { };
+    callback = callback || NOOP_CALLBACK;
 
-    backups.getAllPaged(1, 1, function (error, backups) {
+    backups.getPaged(1, 1, function (error, backups) {
         if (error) {
             debug('Unable to list backups', error);
             return callback(error); // no point trying to backup if appstore is down
@@ -666,7 +646,7 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, function (error, result) {
+    backups.getBackupUrl(appBackupIds, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -703,7 +683,7 @@ function backupBox(callback) {
 
 // this function expects you to have a lock
 function backupBoxAndApps(callback) {
-    callback = callback || function () { }; // callback can be empty for timer triggered backup
+    callback = callback || NOOP_CALLBACK;
 
     apps.getAll(function (error, allApps) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
@@ -734,10 +714,47 @@ function backupBoxAndApps(callback) {
 
             backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
 
-            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
                 progress.set(progress.BACKUP, 100, error ? error.message : '');
-                callback(error, restoreKey);
+
+                callback(error, filename);
             });
         });
     });
 }
+
+function checkDiskSpace(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Checking disk space');
+
+    df(function (error, entries) {
+        if (error) {
+            debug('df error %s', error.message);
+            mailer.outOfDiskSpace(error.message);
+            return callback();
+        }
+
+        var oos = entries.some(function (entry) {
+            return (entry.mount === paths.DATA_DIR && entry.capacity >= 0.90) ||
+                   (entry.mount === '/' && entry.used <= (1.25 * 1024 * 1024)); // 1.5G
+        });
+
+        debug('Disk space checked. ok: %s', !oos);
+
+        if (oos) mailer.outOfDiskSpace(JSON.stringify(entries, null, 4));
+
+        callback();
+    });
+}
+
+function retire(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    var data = {
+        isCustomDomain: config.isCustomDomain(),
+        fqdn: config.fqdn()
+    };
+    shell.sudo('retire', [ RETIRE_CMD, JSON.stringify(data) ], callback);
+}
+

src/config.js

@@ -23,6 +23,7 @@ exports = module.exports = {
     fqdn: fqdn,
     token: token,
     version: version,
+    setVersion: setVersion,
     isCustomDomain: isCustomDomain,
     database: database,
 
@@ -32,11 +33,12 @@ exports = module.exports = {
     adminFqdn: adminFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
+    adminEmail: adminEmail,
 
     isDev: isDev,
 
     // for testing resets to defaults
-    _reset: initConfig
+    _reset: _reset
 };
 
 var assert = require('assert'),
@@ -69,6 +71,14 @@ function saveSync() {
     fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
 
+function _reset (callback) {
+    safe.fs.unlinkSync(cloudronConfigFileName);
+
+    initConfig();
+
+    if (callback) callback();
+}
+
 function initConfig() {
     // setup defaults
     data.fqdn = 'localhost';
@@ -100,6 +110,7 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
+        data.adminEmail = 'test@cloudron.foo';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -138,6 +149,10 @@ function get(key) {
     return safe.query(data, key);
 }
 
+function adminEmail() {
+    return '"Cloudron" ' + get('adminEmail');
+}
+
 function apiServerOrigin() {
     return get('apiServerOrigin');
 }
@@ -178,6 +193,10 @@ function version() {
     return get('version');
 }
 
+function setVersion(version) {
+    set('version', version);
+}
+
 function isCustomDomain() {
     return get('isCustomDomain');
 }

src/constants.js

@@ -7,6 +7,8 @@ exports = module.exports = {
     ADMIN_NAME: 'Settings',
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
-    ADMIN_APPID: 'admin' // admin appid (settingsdb)
+    ADMIN_APPID: 'admin', // admin appid (settingsdb)
+
+    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
 };
 

src/cron.js

@@ -25,7 +25,8 @@ var gAutoupdaterJob = null,
     gCleanupTokensJob = null,
     gDockerVolumeCleanerJob = null,
     gSchedulerSyncJob = null,
-    gCertificateRenewJob = null;
+    gCertificateRenewJob = null,
+    gCheckDiskSpaceJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 
@@ -69,6 +70,14 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCheckDiskSpaceJob) gCheckDiskSpaceJob.stop();
+        gCheckDiskSpaceJob = new CronJob({
+            cronTime: '00 30 */4 * * *', // every 4 hours
+            onTick: cloudron.checkDiskSpace,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
         if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
         gBoxUpdateCheckerJob = new CronJob({
             cronTime: '00 */10 * * * *', // every 10 minutes

src/database.js

@@ -118,6 +118,7 @@ function clear(callback) {
         require('./authcodedb.js')._clear,
         require('./clientdb.js')._clear,
         require('./tokendb.js')._clear,
+        require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
         require('./settingsdb.js')._clear
     ], callback);

src/databaseerror.js

@@ -30,3 +30,4 @@ DatabaseError.INTERNAL_ERROR = 'Internal error';
 DatabaseError.ALREADY_EXISTS = 'Entry already exist';
 DatabaseError.NOT_FOUND = 'Record not found';
 DatabaseError.BAD_FIELD = 'Invalid field';
+DatabaseError.IN_USE = 'In Use';

src/dns/route53.js

@@ -39,7 +39,8 @@ function getZoneByName(dnsConfig, zoneName, callback) {
 
     var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
     route53.listHostedZones({}, function (error, result) {
-        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
 
         var zone = result.HostedZones.filter(function (zone) {
             return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
@@ -84,11 +85,9 @@ function add(dnsConfig, zoneName, subdomain, type, values, callback) {
 
         var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
         route53.changeResourceRecordSets(params, function(error, result) {
-            if (error && error.code === 'PriorRequestNotComplete') {
-                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
-            } else if (error) {
-                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
-            }
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error && error.code === 'PriorRequestNotComplete') return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
 
             callback(null, result.ChangeInfo.Id);
         });
@@ -131,7 +130,8 @@ function get(dnsConfig, zoneName, subdomain, type, callback) {
 
         var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
         route53.listResourceRecordSets(params, function (error, result) {
-            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
             if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
             if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
 
@@ -175,21 +175,22 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
 
         var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
         route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
             if (error && error.message && error.message.indexOf('it was not found') !== -1) {
                 debug('del: resource record set not found.', error);
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
             } else if (error && error.code === 'NoSuchHostedZone') {
                 debug('del: hosted zone not found.', error);
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
             } else if (error && error.code === 'PriorRequestNotComplete') {
                 debug('del: resource is still busy', error);
-                return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
             } else if (error && error.code === 'InvalidChangeBatch') {
                 debug('del: invalid change batch. No such record to be deleted.');
-                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
             } else if (error) {
                 debug('del: error', error);
-                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
             }
 
             callback(null);
@@ -206,6 +207,7 @@ function getChangeStatus(dnsConfig, changeId, callback) {
 
     var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
     route53.getChange({ Id: changeId }, function (error, result) {
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
         if (error) return callback(error);
 
         callback(null, result.ChangeInfo.Status);

src/docker.js

@@ -4,6 +4,7 @@ var addons = require('./addons.js'),
     async = require('async'),
     assert = require('assert'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:src/docker.js'),
     Docker = require('dockerode'),
     safe = require('safetydance'),
@@ -23,7 +24,8 @@ exports = module.exports = {
     deleteContainerByName: deleteContainer,
     deleteImage: deleteImage,
     deleteContainers: deleteContainers,
-    createSubcontainer: createSubcontainer
+    createSubcontainer: createSubcontainer,
+    getContainerIdByIp: getContainerIdByIp
 };
 
 function connectionInstance() {
@@ -156,7 +158,15 @@ function createSubcontainer(app, name, cmd, options, callback) {
         dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
     }
 
-    var memoryLimit = manifest.memoryLimit || (developmentMode ? 0 : 1024 * 1024 * 200); // 200mb by default
+    // first check db record, then manifest
+    var memoryLimit = app.memoryLimit || manifest.memoryLimit;
+
+    // ensure we never go below minimum
+    memoryLimit = memoryLimit < constants.DEFAULT_MEMORY_LIMIT ? constants.DEFAULT_MEMORY_LIMIT : memoryLimit; // 256mb by default
+
+    // developerMode does not restrict memory usage
+    memoryLimit = developmentMode ? 0 : memoryLimit;
+
     // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
     // this means cloudron exec does not work
     var isolatedNetworkNs = true;
@@ -171,7 +181,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
             Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
             Tty: isAppContainer,
             Image: app.manifest.dockerImage,
-            Cmd: (isAppContainer && developmentMode) ? [ '/bin/sleep', 'infinity' ] : cmd,
+            Cmd: (isAppContainer && developmentMode) ? [ '/bin/bash', '-c', 'echo "Development mode. Use cloudron exec to debug. Sleeping" && sleep infinity' ] : cmd,
             Env: stdEnv.concat(addonEnv).concat(portEnv),
             ExposedPorts: isAppContainer ? exposedPorts : { },
             Volumes: { // see also ReadonlyRootfs
@@ -329,24 +339,53 @@ function deleteImage(manifest, callback) {
 
     var docker = exports.connection;
 
-    docker.getImage(dockerImage).inspect(function (error, result) {
+    var removeOptions = {
+        force: false, // might be shared with another instance of this app
+        noprune: false // delete untagged parents
+    };
+
+    // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
+    // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
+    // not created anymore after https://github.com/docker/docker/pull/10571
+    docker.getImage(dockerImage).remove(removeOptions, function (error) {
         if (error && error.statusCode === 404) return callback(null);
+        if (error && error.statusCode === 409) return callback(null); // another container using the image
 
-        if (error) return callback(error);
+        if (error) debug('Error removing image %s : %j', dockerImage, error);
 
-        var removeOptions = {
-            force: true,
-            noprune: false
-        };
-
-         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
-        docker.getImage(result.Id).remove(removeOptions, function (error) {
-            if (error && error.statusCode === 404) return callback(null);
-            if (error && error.statusCode === 409) return callback(null); // another container using the image
-
-            if (error) debug('Error removing image %s : %j', dockerImage, error);
-
-            callback(error);
-        });
+        callback(error);
+    });
+}
+
+function getContainerIdByIp(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('get container by ip %s', ip);
+
+    var docker = exports.connection;
+
+    docker.listNetworks({}, function (error, result) {
+        if (error) return callback(error);
+
+        var bridge;
+        result.forEach(function (n) {
+            if (n.Name === 'bridge') bridge = n;
+        });
+
+        if (!bridge) return callback(new Error('Unable to find the bridge network'));
+
+        var containerId;
+        for (var id in bridge.Containers) {
+            if (bridge.Containers[id].IPv4Address.indexOf(ip) === 0) {
+                containerId = id;
+                break;
+            }
+        }
+        if (!containerId) return callback(new Error('No container with that ip'));
+
+        debug('found container %s with ip %s', containerId, ip);
+
+        callback(null, containerId);
     });
 }

src/groupdb.js

@@ -0,0 +1,202 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+    add: add,
+    del: del,
+    count: count,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror');
+
+var GROUPS_FIELDS = [ 'id', 'name' ].join(',');
+
+function get(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups WHERE id = ?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result[0]);
+    });
+}
+
+function getWithMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' WHERE groups.id = ? ' +
+                    ' GROUP BY groups.id', [ groupId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        var result = results[0];
+        result.userIds = result.userIds ? result.userIds.split(',') : [ ];
+
+        callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function add(id, name, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [ id, name ];
+    database.query('INSERT INTO groups (id, name) VALUES (?, ?)',
+           data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
+    queries.push({ query: 'DELETE FROM groups WHERE id = ?', args: [ id ] });
+
+    database.transaction(queries, function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(error);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM groupMembers', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        database.query('DELETE FROM groups WHERE id != ?', [ 'admin' ], function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            callback(error);
+        });
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.userId; }));
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.groupId; }));
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [ ];
+    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
+    groupIds.forEach(function (gid) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result.length !== 0);
+    });
+}

src/groups.js

@@ -0,0 +1,210 @@
+/* jshint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    GroupError: GroupError,
+
+    create: create,
+    remove: remove,
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    ADMIN_GROUP_ID: 'admin' // see db migration code and groupdb._clear
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    groupdb = require('./groupdb.js'),
+    util = require('util');
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function GroupError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(GroupError, Error);
+GroupError.INTERNAL_ERROR = 'Internal Error';
+GroupError.ALREADY_EXISTS = 'Already Exists';
+GroupError.NOT_FOUND = 'Not Found';
+GroupError.BAD_NAME = 'Bad name';
+GroupError.NOT_EMPTY = 'Not Empty';
+GroupError.NOT_ALLOWED = 'Not Allowed';
+
+function validateGroupname(name) {
+    assert.strictEqual(typeof name, 'string');
+    var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
+
+    if (name.length <= 2) return new GroupError(GroupError.BAD_NAME, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_NAME, 'name too long');
+
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_NAME, 'name can only have A-Za-z0-9_-');
+
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_NAME, 'name is reserved');
+
+    return null;
+}
+
+function create(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateGroupname(name);
+    if (error) return callback(error);
+
+    groupdb.add(name /* id */, name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new GroupError(GroupError.ALREADY_EXISTS));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null, { id: name, name: name });
+    });
+}
+
+function remove(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // never allow admin group to be deleted
+    if (id === exports.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
+
+    groupdb.del(id, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getWithMembers(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getWithMembers(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAll(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getMembers(groupId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getGroups(userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.setGroups(userId, groupIds, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.addMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.removeMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.isMember(groupId, userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}

src/ldap.js

@@ -6,6 +6,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    apps = require('./apps.js'),
     config = require('./config.js'),
     debug = require('debug')('box:ldap'),
     user = require('./user.js'),
@@ -28,15 +29,25 @@ var gLogger = {
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
 var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
+function getAppByRequest(req, callback) {
+    var sourceIp = req.connection.ldap.id.split(':')[0];
+    if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
+
+    apps.getByIpAddress(sourceIp, function (error, app) {
+        // we currently allow access in case we can't find the source app
+        callback(null, app || null);
+    });
+}
+
 function start(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     gServer = ldap.createServer({ log: gLogger });
 
     gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+        debug('user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
 
-        user.list(function (error, result){
+        user.list(function (error, result) {
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             // send user objects
@@ -54,7 +65,7 @@ function start(callback) {
                         cn: entry.id,
                         uid: entry.id,
                         mail: entry.email,
-                        displayname: entry.username,
+                        displayname: entry.displayName || entry.username,
                         username: entry.username,
                         samaccountname: entry.username,      // to support ActiveDirectory clients
                         memberof: groups
@@ -71,7 +82,7 @@ function start(callback) {
     });
 
     gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+        debug('group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
 
         user.list(function (error, result){
             if (error) return next(new ldap.OperationsError(error.toString()));
@@ -108,21 +119,44 @@ function start(callback) {
 
     gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
         // TODO: validate password
-        debug('ldap application bind: %s', req.dn.toString());
+        debug('application bind: %s', req.dn.toString());
         res.end();
     });
 
     gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('ldap user bind: %s', req.dn.toString());
+        debug('user bind: %s', req.dn.toString());
 
-        if (!req.dn.rdns[0].cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        // extract the common name which might have different attribute names
+        var attributeName = Object.keys(req.dn.rdns[0])[0];
+        var commonName = req.dn.rdns[0][attributeName];
+        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-        user.verify(req.dn.rdns[0].cn, req.credentials || '', function (error, result) {
+        // if mail is specified, enforce mail check, otherwise allow both
+        var api = (commonName.indexOf('@') === -1) && (attributeName !== 'mail')  ? user.verify : user.verifyWithEmail;
+
+        // TODO this should be done after we verified the app has access to avoid leakage of user existence
+        api(commonName, req.credentials || '', function (error, userObject) {
             if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error));
 
-            res.end();
+            getAppByRequest(req, function (error, app) {
+                if (error) return next(error);
+
+                if (!app) {
+                    debug('no app found for this container, allow access');
+                    return res.end();
+                }
+
+                apps.hasAccessTo(app, userObject, function (error, result) {
+                    if (error) return next(new ldap.OperationsError(error.toString()));
+
+                    // we return no such object, to avoid leakage of a users existence
+                    if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+                    res.end();
+                });
+            });
         });
     });
 

src/mail_templates/app_update_available.ejs

@@ -2,10 +2,13 @@
 
 Dear Admin,
 
-A new version of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+A new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
 
 The app will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
 
+Changes:
+<%= updateInfo.manifest.changelog %>
+
 Thank you,
 your Cloudron
 

src/mail_templates/box_update_available.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-A new version of Cloudron <%= fqdn %> is available!
+Version <%= newBoxVersion %> of Cloudron <%= fqdn %> is now available!
 
 Your Cloudron will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
 

src/mail_templates/certificate_renewed.ejs

@@ -0,0 +1,16 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+<% if (message) { %>
+<%= domain %> was not renewed.
+
+<%- message %>
+<% } else { %>
+<%= domain %> was renewed.
+<% } %>
+Thank you,
+Your Cloudron
+<% } else { %>
+
+<% } %>
+

src/mail_templates/out_of_disk_space.ejs

@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+
+<%= fqdn %> is running out of disk space.
+
+Please see some excerpts of the logs below.
+
+Thank you,
+Your Cloudron
+
+-------------------------------------
+
+<%- message %>
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/user_added.ejs

@@ -0,0 +1,23 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+User with name '<%= username %>' (<%= email %>) was added in the Cloudron at <%= fqdn %>.
+
+You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
+
+<% if (inviteLink) { %>
+As requested, this user has not been sent an invitation email.
+
+To set a password and perform any configuration on behalf of the user, please use this link:
+<%= inviteLink %>
+
+<% } %>
+
+Thank you,
+User Manager
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/welcome_user.ejs

@@ -2,7 +2,7 @@
 
 Dear <%= user.username %>,
 
-Welcome to my Cloudron <%= fqdn %>!
+Welcome to our Cloudron <%= fqdn %>!
 
 The Cloudron is our own Smart Server. You can read more about it
 at https://www.cloudron.io.
@@ -15,8 +15,12 @@ To get started, create your account by visiting the following page:
 When you visit the above page, you will be prompted to enter a new password.
 After you have submitted the form, you can login using the new password.
 
+<% if (invitor && invitor.email) { %>
 Thank you,
 <%= invitor.email %>
+<% } else { %>
+Thank you
+<% } %>
 
 <% } else { %>
 

src/mailer.js

@@ -13,14 +13,23 @@ exports = module.exports = {
     boxUpdateAvailable: boxUpdateAvailable,
     appUpdateAvailable: appUpdateAvailable,
 
+    sendInvite: sendInvite,
     sendCrashNotification: sendCrashNotification,
 
     appDied: appDied,
 
+    outOfDiskSpace: outOfDiskSpace,
+
+    certificateRenewed: certificateRenewed,
+
     FEEDBACK_TYPE_FEEDBACK: 'feedback',
     FEEDBACK_TYPE_TICKET: 'ticket',
-    FEEDBACK_TYPE_APP: 'app',
-    sendFeedback: sendFeedback
+    FEEDBACK_TYPE_APP_MISSING: 'app_missing',
+    FEEDBACK_TYPE_APP_ERROR: 'app_error',
+    sendFeedback: sendFeedback,
+
+    _getMailQueue: _getMailQueue,
+    _clearMailQueue: _clearMailQueue
 };
 
 var assert = require('assert'),
@@ -170,6 +179,9 @@ function sendMails(queue) {
 function enqueue(mailOptions) {
     assert.strictEqual(typeof mailOptions, 'object');
 
+    if (!mailOptions.from) console.error('sender address is missing');
+    if (!mailOptions.to) console.error('recipient address is missing');
+
     debug('Queued mail for ' + mailOptions.from + ' to ' + mailOptions.to);
     gMailQueue.push(mailOptions);
 
@@ -204,7 +216,7 @@ function mailUserEventToAdmins(user, event) {
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: config.adminEmail(),
             to: adminEmails.join(', '),
             subject: util.format('%s %s in Cloudron %s', user.username, event, config.fqdn()),
             text: render('user_event.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, event: event, format: 'text' }),
@@ -214,11 +226,11 @@ function mailUserEventToAdmins(user, event) {
     });
 }
 
-function userAdded(user, invitor) {
+function sendInvite(user, invitor) {
     assert.strictEqual(typeof user, 'object');
     assert(typeof invitor === 'object');
 
-    debug('Sending mail for userAdded');
+    debug('Sending invite mail');
 
     var templateData = {
         user: user,
@@ -230,15 +242,37 @@ function userAdded(user, invitor) {
     };
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: config.adminEmail(),
         to: user.email,
         subject: util.format('Welcome to Cloudron %s', config.fqdn()),
         text: render('welcome_user.ejs', templateData)
     };
 
     enqueue(mailOptions);
+}
 
-    mailUserEventToAdmins(user, 'was added');
+function userAdded(user, inviteSent) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof inviteSent, 'boolean');
+
+    debug('Sending mail for userAdded %s including invite link', inviteSent ? 'not' : '');
+
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
+
+        adminEmails = _.difference(adminEmails, [ user.email ]);
+
+        var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken;
+
+        var mailOptions = {
+            from: config.adminEmail(),
+            to: adminEmails.join(', '),
+            subject: util.format('%s added in Cloudron %s', user.username, config.fqdn()),
+            text: render('user_added.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, inviteLink: inviteLink, format: 'text' }),
+        };
+
+        enqueue(mailOptions);
+    });
 }
 
 function userRemoved(username) {
@@ -249,12 +283,13 @@ function userRemoved(username) {
     mailUserEventToAdmins({ username: username }, 'was removed');
 }
 
-function adminChanged(user) {
+function adminChanged(user, admin) {
     assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof admin, 'boolean');
 
     debug('Sending mail for adminChanged');
 
-    mailUserEventToAdmins(user, user.admin ? 'is now an admin' : 'is no more an admin');
+    mailUserEventToAdmins(user, admin ? 'is now an admin' : 'is no more an admin');
 }
 
 function passwordReset(user) {
@@ -265,7 +300,7 @@ function passwordReset(user) {
     var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: config.adminEmail(),
         to: user.email,
         subject: 'Password Reset Request',
         text: render('password_reset.ejs', { fqdn: config.fqdn(), username: user.username, resetLink: resetLink, format: 'text' })
@@ -283,7 +318,7 @@ function appDied(app) {
         if (error) return console.log('Error getting admins', error);
 
         var mailOptions = {
-            from: config.get('adminEmail'),
+            from: config.adminEmail(),
             to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
@@ -301,7 +336,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.get('adminEmail'),
+            from: config.adminEmail(),
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', config.fqdn()),
             text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
@@ -319,16 +354,43 @@ function appUpdateAvailable(app, updateInfo) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.get('adminEmail'),
+            from: config.adminEmail(),
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', app.fqdn),
-            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, format: 'text' })
+            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
         };
 
         enqueue(mailOptions);
     });
 }
 
+function outOfDiskSpace(message) {
+    assert.strictEqual(typeof message, 'string');
+
+    var mailOptions = {
+        from: config.adminEmail(),
+        to: 'admin@cloudron.io',
+        subject: util.format('[%s] Out of disk space alert', config.fqdn()),
+        text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
+    };
+
+    sendMails([ mailOptions ]);
+}
+
+function certificateRenewed(domain, message) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof message, 'string');
+
+    var mailOptions = {
+        from: config.adminEmail(),
+        to: 'admin@cloudron.io',
+        subject: util.format('[%s] Certificate was %s renewed', domain, message ? 'not' : ''),
+        text: render('certificate_renewed.ejs', { domain: domain, message: message, format: 'text' })
+    };
+
+    sendMails([ mailOptions ]);
+}
+
 // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
 // crashnotifier should be able to send mail when there is no db
 function sendCrashNotification(program, context) {
@@ -336,7 +398,7 @@ function sendCrashNotification(program, context) {
     assert.strictEqual(typeof context, 'string');
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: config.adminEmail(),
         to: 'admin@cloudron.io',
         subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
         text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
@@ -351,10 +413,13 @@ function sendFeedback(user, type, subject, description) {
     assert.strictEqual(typeof subject, 'string');
     assert.strictEqual(typeof description, 'string');
 
-    assert(type === exports.FEEDBACK_TYPE_TICKET || type === exports.FEEDBACK_TYPE_FEEDBACK || type === exports.FEEDBACK_TYPE_APP);
+    assert(type === exports.FEEDBACK_TYPE_TICKET ||
+        type === exports.FEEDBACK_TYPE_FEEDBACK ||
+        type === exports.FEEDBACK_TYPE_APP_MISSING ||
+        type === exports.FEEDBACK_TYPE_APP_ERROR);
 
     var mailOptions = {
-        from: config.get('adminEmail'),
+        from: config.adminEmail(),
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})
@@ -362,3 +427,13 @@ function sendFeedback(user, type, subject, description) {
 
     enqueue(mailOptions);
 }
+
+function _getMailQueue() {
+    return gMailQueue;
+}
+
+function _clearMailQueue(callback) {
+    gMailQueue = [];
+
+    if (callback) callback();
+}

src/nginx.js

@@ -13,6 +13,7 @@ var assert = require('assert'),
     shell = require('./shell.js');
 
 exports = module.exports = {
+    requiresOAuthProxy: requiresOAuthProxy,
     configureAdmin: configureAdmin,
     configureApp: configureApp,
     unconfigureApp: unconfigureApp,
@@ -22,6 +23,19 @@ exports = module.exports = {
 var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
     RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
 
+function requiresOAuthProxy(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    var tmp = app.accessRestriction;
+
+    // if no accessRestriction set, or the app uses one of the auth modules, we do not need the oauth proxy
+    if (tmp === null) return false;
+    if (app.manifest.addons['ldap'] || app.manifest.addons['oauth'] || app.manifest.addons['simpleauth']) return false;
+
+    // check if any restrictions are set
+    return !!((tmp.users && tmp.users.length) || (tmp.groups && tmp.groups.length));
+}
+
 function configureAdmin(certFilePath, keyFilePath, callback) {
     assert.strictEqual(typeof certFilePath, 'string');
     assert.strictEqual(typeof keyFilePath, 'string');
@@ -50,7 +64,8 @@ function configureApp(app, certFilePath, keyFilePath, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var sourceDir = path.resolve(__dirname, '..');
-    var endpoint = app.oauthProxy ? 'oauthproxy' : 'app';
+    var oauthProxy = requiresOAuthProxy(app);
+    var endpoint = oauthProxy ? 'oauthproxy' : 'app';
     var vhost = config.appFqdn(app.location);
 
     var data = {

src/oauth2views/header.ejs

@@ -4,7 +4,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
 
-    <title> Cloudron Login </title>
+    <title> <%= title %> </title>
 
     <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
 

src/oauth2views/password_reset.ejs

@@ -25,10 +25,16 @@ app.controller('Controller', [function () {}]);
 
                 <div class="form-group" ng-class="{ 'has-error': resetForm.password.$dirty && resetForm.password.$invalid }">
                     <label class="control-label" for="inputPassword">New Password</label>
-                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-maxlength="512" ng-minlength="5" autofocus required>
+                    <div class="control-label" ng-show="resetForm.password.$dirty && resetForm.password.$invalid">
+                        <small ng-show="resetForm.password.$dirty && resetForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+                    </div>
+                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" autofocus required>
                 </div>
-                <div class="form-group" ng-class="{ 'has-error': resetForm.passwordRepeat.$dirty && (resetForm.passwordRepeat.$invalid || password !== passwordRepeat) }">
+                <div class="form-group" ng-class="{ 'has-error': resetForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
                     <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
+                    <div class="control-label" ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
+                        <small ng-show="resetForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
+                    </div>
                     <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
                 </div>
                 <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="resetForm.$invalid || password !== passwordRepeat"/>

src/oauth2views/password_setup.ejs

@@ -13,7 +13,7 @@ app.controller('Controller', [function () {}]);
 </script>
 
 <center>
-    <h1>Hello <%= user.username %> create a password</h1>
+    <h1>Hello <%= user.username %>, set a password</h1>
 </center>
 
 <div class="container" ng-app="Application" ng-controller="Controller">
@@ -25,10 +25,16 @@ app.controller('Controller', [function () {}]);
 
                 <div class="form-group" ng-class="{ 'has-error': setupForm.password.$dirty && setupForm.password.$invalid }">
                     <label class="control-label" for="inputPassword">New Password</label>
-                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-maxlength="512" ng-minlength="5" autofocus required>
+                    <div class="control-label" ng-show="setupForm.password.$dirty && setupForm.password.$invalid">
+                        <small ng-show="setupForm.password.$dirty && setupForm.password.$invalid">Password must be 8-30 character with at least one uppercase, one numeric and one special character</small>
+                    </div>
+                    <input type="password" class="form-control" id="inputPassword" ng-model="password" name="password" ng-pattern="/^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/" autofocus required>
                 </div>
-                <div class="form-group" ng-class="{ 'has-error': setupForm.passwordRepeat.$dirty && (setupForm.passwordRepeat.$invalid || password !== passwordRepeat) }">
+                <div class="form-group" ng-class="{ 'has-error': setupForm.passwordRepeat.$dirty && (password !== passwordRepeat) }">
                     <label class="control-label" for="inputPasswordRepeat">Repeat Password</label>
+                    <div class="control-label" ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">
+                        <small ng-show="setupForm.passwordRepeat.$dirty && (password !== passwordRepeat)">Passwords don't match</small>
+                    </div>
                     <input type="password" class="form-control" id="inputPasswordRepeat" ng-model="passwordRepeat" name="passwordRepeat" required>
                 </div>
                 <input class="btn btn-primary btn-outline pull-right" type="submit" value="Create" ng-disabled="setupForm.$invalid || password !== passwordRepeat"/>

src/oauthproxy.js

@@ -126,7 +126,7 @@ function authenticate(req, res, next) {
 
             clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
                 if (error) {
-                    console.error('Unkonwn OAuth client.', error);
+                    console.error('Unknown OAuth client.', error);
                     return res.send(500, 'Unknown OAuth client.');
                 }
 

src/password.js

@@ -0,0 +1,47 @@
+/* jslint node:true */
+
+'use strict';
+
+// From https://www.npmjs.com/package/password-generator
+
+exports = module.exports = {
+    generate: generate,
+    validate: validate
+};
+
+var assert = require('assert'),
+    generatePassword = require('password-generator');
+
+// http://www.w3resource.com/javascript/form/example4-javascript-form-validation-password.html
+// WARNING!!! if this is changed, the UI parts in the setup and account view have to be adjusted!
+var gPasswordTestRegExp = /^(?=.*\d)(?=.*[a-z])(?=.*[A-Z])(?=.*[^a-zA-Z0-9])(?!.*\s).{8,30}$/;
+
+var UPPERCASE_RE = /([A-Z])/g;
+var LOWERCASE_RE = /([a-z])/g;
+var NUMBER_RE = /([\d])/g;
+var SPECIAL_CHAR_RE = /([\?\-])/g;
+
+function isStrongEnough(password) {
+    var uc = password.match(UPPERCASE_RE);
+    var lc = password.match(LOWERCASE_RE);
+    var n = password.match(NUMBER_RE);
+    var sc = password.match(SPECIAL_CHAR_RE);
+
+    return uc && lc && n && sc;
+}
+
+function generate() {
+    var password = '';
+
+    while (!isStrongEnough(password)) password = generatePassword(8, false, /[\w\d\?\-]/);
+
+    return password;
+}
+
+function validate(password) {
+    assert.strictEqual(typeof password, 'string');
+
+    if (!password.match(gPasswordTestRegExp)) return new Error('Password must be 8-30 character with at least one uppercase, one numeric and one special character');
+
+    return null;
+}

src/routes/apps.js

@@ -15,6 +15,7 @@ exports = module.exports = {
     updateApp: updateApp,
     getLogs: getLogs,
     getLogStream: getLogStream,
+    listBackups: listBackups,
 
     stopApp: stopApp,
     startApp: startApp,
@@ -43,12 +44,12 @@ function removeInternalAppFields(app) {
         health: app.health,
         location: app.location,
         accessRestriction: app.accessRestriction,
-        oauthProxy: app.oauthProxy,
         lastBackupId: app.lastBackupId,
         manifest: app.manifest,
         portBindings: app.portBindings,
         iconUrl: app.iconUrl,
-        fqdn: app.fqdn
+        fqdn: app.fqdn,
+        memoryLimit: app.memoryLimit
     };
 }
 
@@ -75,7 +76,10 @@ function getAppBySubdomain(req, res, next) {
 }
 
 function getApps(req, res, next) {
-    apps.getAll(function (error, allApps) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    var func = req.user.admin ? apps.getAll : apps.getAllByUser.bind(null, req.user);
+    func(function (error, allApps) {
         if (error) return next(new HttpError(500, error));
 
         allApps = allApps.map(removeInternalAppFields);
@@ -115,19 +119,19 @@ function installApp(req, res, next) {
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
     if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (typeof data.oauthProxy !== 'boolean') return next(new HttpError(400, 'oauthProxy must be a boolean'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
     if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+    if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
 
     // allow tests to provide an appId for testing
     var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
 
-    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j oauthproxy:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.oauthProxy, data.manifest);
+    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j memoryLimit:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.memoryLimit, data.manifest);
 
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.oauthProxy, data.icon || null, data.cert || null, data.key || null, function (error) {
+    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -159,15 +163,15 @@ function configureApp(req, res, next) {
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
     if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (typeof data.oauthProxy !== 'boolean') return next(new HttpError(400, 'oauthProxy must be a boolean'));
     if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
     if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+    if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
 
-    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j oauthProxy:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.oauthProxy);
+    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j memoryLimit:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.memoryLimit);
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.oauthProxy, data.cert || null, data.key || null, function (error) {
+    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -357,7 +361,9 @@ function exec(req, res, next) {
     var rows = req.query.rows ? parseInt(req.query.rows, 10) : null;
     if (isNaN(rows)) return next(new HttpError(400, 'rows must be a number'));
 
-    apps.exec(req.params.id, { cmd: cmd, rows: rows, columns: columns }, function (error, duplexStream) {
+    var tty = req.query.tty === 'true' ? true : false;
+
+    apps.exec(req.params.id, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
@@ -371,3 +377,20 @@ function exec(req, res, next) {
         res.socket.pipe(duplexStream);
     });
 }
+
+function listBackups(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    apps.listBackups(page, perPage, req.params.id, function (error, result) {
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { backups: result }));
+    });
+}

src/routes/backups.js

@@ -4,19 +4,26 @@
 
 exports = module.exports = {
     get: get,
-    create: create
+    create: create,
+    download: download
 };
 
-var backups = require('../backups.js'),
+var assert = require('assert'),
+    backups = require('../backups.js'),
     BackupsError = require('../backups.js').BackupsError,
     cloudron = require('../cloudron.js'),
     CloudronError = require('../cloudron.js').CloudronError,
-    debug = require('debug')('box:routes/backups'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
 
 function get(req, res, next) {
-    backups.getAllPaged(1, 5, function (error, result) {
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
+
+    backups.getPaged(page, perPage, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -34,3 +41,14 @@ function create(req, res, next) {
         next(new HttpSuccess(202, {}));
     });
 }
+
+function download(req, res, next) {
+    assert.strictEqual(typeof req.params.backupId, 'string');
+
+    backups.getRestoreUrl(req.params.backupId, function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+

src/routes/cloudron.js

@@ -10,7 +10,6 @@ exports = module.exports = {
     getProgress: getProgress,
     getConfig: getConfig,
     update: update,
-    migrate: migrate,
     feedback: feedback
 };
 
@@ -41,15 +40,17 @@ function activate(req, res, next) {
     if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be string'));
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be string'));
     if (typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
 
     var username = req.body.username;
     var password = req.body.password;
     var email = req.body.email;
+    var displayName = req.body.displayName || '';
 
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
     debug('activate: username:%s ip:%s', username, ip);
 
-    cloudron.activate(username, password, email, ip, function (error, info) {
+    cloudron.activate(username, password, email, displayName, ip, function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
         if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
         if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
@@ -127,24 +128,13 @@ function update(req, res, next) {
     });
 }
 
-function migrate(req, res, next) {
-    if (typeof req.body.size !== 'string') return next(new HttpError(400, 'size must be string'));
-    if (typeof req.body.region !== 'string') return next(new HttpError(400, 'region must be string'));
-
-    debug('Migration requested', req.body.size, req.body.region);
-
-    cloudron.migrate(req.body.size, req.body.region, function (error) {
-        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(202, {}));
-    });
-}
-
 function feedback(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK && req.body.type !== mailer.FEEDBACK_TYPE_TICKET && req.body.type !== mailer.FEEDBACK_TYPE_APP) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app"'));
+    if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK &&
+        req.body.type !== mailer.FEEDBACK_TYPE_TICKET &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_MISSING &&
+        req.body.type !== mailer.FEEDBACK_TYPE_APP_ERROR) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app_missing" or "app_error"'));
     if (typeof req.body.subject !== 'string' || !req.body.subject) return next(new HttpError(400, 'subject must be string'));
     if (typeof req.body.description !== 'string' || !req.body.description) return next(new HttpError(400, 'description must be string'));
 

src/routes/groups.js

@@ -0,0 +1,67 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    list: list,
+    create: create,
+    remove: remove
+};
+
+var assert = require('assert'),
+    groups = require('../groups.js'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    groups = require('../groups.js'),
+    GroupError = groups.GroupError;
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    groups.create(req.body.name, function (error, group) {
+        if (error && error.reason === GroupError.BAD_NAME) return next(new HttpError(400, error.message));
+        if (error && error.reason === GroupError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        var groupInfo = {
+            id: group.id,
+            name: group.name
+        };
+
+        next(new HttpSuccess(201, groupInfo));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.groupId, 'string');
+
+    groups.getWithMembers(req.params.groupId, function (error, result) {
+        if (error && error.reason === GroupError.NOT_FOUND) return next(new HttpError(404, 'No such group'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    groups.getAll(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { groups: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.groupId, 'string');
+
+    groups.remove(req.params.groupId, function (error) {
+        if (error && error.reason === GroupError.NOT_FOUND) return next(new HttpError(404, 'Group not found'));
+        if (error && error.reason === GroupError.NOT_ALLOWED) return next(new HttpError(409, 'Group deletion not allowed'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/index.js

@@ -2,14 +2,14 @@
 
 exports = module.exports = {
     apps: require('./apps.js'),
+    backups: require('./backups.js'),
+    clients: require('./clients.js'),
     cloudron: require('./cloudron.js'),
     developer: require('./developer.js'),
     graphs: require('./graphs.js'),
+    groups: require('./groups.js'),
+    internal: require('./internal.js'),
     oauth2: require('./oauth2.js'),
     settings: require('./settings.js'),
-    clients: require('./clients.js'),
-    backups: require('./backups.js'),
-    internal: require('./internal.js'),
     user: require('./user.js')
 };
-

src/routes/internal.js

@@ -4,7 +4,8 @@
 
 exports = module.exports = {
     backup: backup,
-    update: update
+    update: update,
+    retire: retire
 };
 
 var cloudron = require('../cloudron.js'),
@@ -39,3 +40,15 @@ function update(req, res, next) {
     });
 }
 
+function retire(req, res, next) {
+    debug('triggering retire');
+
+    // note that cloudron.backup only waits for backup initiation and not for backup to complete
+    // backup progress can be checked up ny polling the progress api call
+    cloudron.retire(function (error) {
+        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}

src/routes/oauth2.js

@@ -150,14 +150,16 @@ function sendErrorPageOrRedirect(req, res, message) {
     if (typeof req.query.returnTo !== 'string') {
         renderTemplate(res, 'error', {
             adminOrigin: config.adminOrigin(),
-            message: message
+            message: message,
+            title: 'Cloudron Error'
         });
     } else {
         var u = url.parse(req.query.returnTo);
         if (!u.protocol || !u.host) {
             return renderTemplate(res, 'error', {
                 adminOrigin: config.adminOrigin(),
-                message: 'Invalid request. returnTo query is not a valid URI. ' + message
+                message: 'Invalid request. returnTo query is not a valid URI. ' + message,
+                title: 'Cloudron Error'
             });
         }
 
@@ -174,7 +176,8 @@ function sendError(req, res, message) {
 
     renderTemplate(res, 'error', {
         adminOrigin: config.adminOrigin(),
-        message: message
+        message: message,
+        title: 'Cloudron Error'
     });
 }
 
@@ -191,7 +194,8 @@ function loginForm(req, res) {
             csrf: req.csrfToken(),
             applicationName: applicationName,
             applicationLogo: applicationLogo,
-            error: req.query.error || null
+            error: req.query.error || null,
+            title: applicationName + ' Login'
         });
     }
 
@@ -237,7 +241,7 @@ function logout(req, res) {
 // Form to enter email address to send a password reset request mail
 // -> GET /api/v1/session/password/resetRequest.html
 function passwordResetRequestSite(req, res) {
-    renderTemplate(res, 'password_reset_request', { adminOrigin: config.adminOrigin(), csrf: req.csrfToken() });
+    renderTemplate(res, 'password_reset_request', { adminOrigin: config.adminOrigin(), csrf: req.csrfToken(), title: 'Cloudron Password Reset' });
 }
 
 // This route is used for above form submission
@@ -261,7 +265,7 @@ function passwordResetRequest(req, res, next) {
 
 // -> GET /api/v1/session/password/sent.html
 function passwordSentSite(req, res) {
-    renderTemplate(res, 'password_reset_sent', { adminOrigin: config.adminOrigin() });
+    renderTemplate(res, 'password_reset_sent', { adminOrigin: config.adminOrigin(), title: 'Cloudron Password Reset' });
 }
 
 // -> GET /api/v1/session/password/setup.html
@@ -275,7 +279,8 @@ function passwordSetupSite(req, res, next) {
             adminOrigin: config.adminOrigin(),
             user: user,
             csrf: req.csrfToken(),
-            resetToken: req.query.reset_token
+            resetToken: req.query.reset_token,
+            title: 'Cloudron Password Setup'
         });
     });
 }
@@ -291,7 +296,8 @@ function passwordResetSite(req, res, next) {
             adminOrigin: config.adminOrigin(),
             user: user,
             csrf: req.csrfToken(),
-            resetToken: req.query.reset_token
+            resetToken: req.query.reset_token,
+            title: 'Cloudron Password Reset'
         });
     });
 }
@@ -310,6 +316,7 @@ function passwordReset(req, res, next) {
 
         // setPassword clears the resetToken
         user.setPassword(userObject.id, req.body.password, function (error, result) {
+            if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
             if (error) return next(new HttpError(500, error));
 
             res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -368,14 +375,17 @@ var authorization = [
 
         if (type === clientdb.TYPE_ADMIN) return next();
         if (type === clientdb.TYPE_EXTERNAL) return next();
-        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unkonwn OAuth client.');
+        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');
 
         appdb.get(req.oauth2.client.appId, function (error, appObject) {
             if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
 
-            if (!apps.hasAccessTo(appObject, req.oauth2.user)) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
+            apps.hasAccessTo(appObject, req.oauth2.user, function (error, access) {
+                if (error) return sendError(req, res, 'Internal error');
+                if (!access) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
 
-            next();
+                next();
+            });
         });
     },
     gServer.decision({ loadTransaction: false })

src/routes/test/backups-test.js

@@ -19,15 +19,17 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;
 
 var server;
 function setup(done) {
+    config.setVersion('1.2.3');
+
     async.series([
         server.start.bind(server),
 
-        userdb._clear,
+        database._clear,
 
         function createAdmin(callback) {
             var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
@@ -51,7 +53,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, false /* oauthProxy */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
         },
 
         function createSettings(callback) {
@@ -72,35 +74,6 @@ describe('Backups API', function () {
     before(setup);
     after(cleanup);
 
-    describe('get', function () {
-        it('cannot get backups with appstore superagent failing', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(401, {});
-
-            superagent.get(SERVER_URL + '/api/v1/backups')
-                   .query({ access_token: token })
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(503);
-                expect(req.isDone()).to.be.ok();
-                done();
-            });
-        });
-
-        it('can get backups', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(200, { backups: ['foo', 'bar']});
-
-            superagent.get(SERVER_URL + '/api/v1/backups')
-                   .query({ access_token: token })
-                   .end(function (err, res) {
-                expect(res.statusCode).to.equal(200);
-                expect(req.isDone()).to.be.ok();
-                expect(res.body.backups).to.be.an(Array);
-                expect(res.body.backups[0]).to.eql('foo');
-                expect(res.body.backups[1]).to.eql('bar');
-                done();
-            });
-        });
-    });
-
     describe('create', function () {
         it('fails due to mising token', function (done) {
             superagent.post(SERVER_URL + '/api/v1/backups')

src/routes/test/clients-test.js

@@ -20,7 +20,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 function cleanup(done) {
@@ -392,7 +392,7 @@ describe('Clients', function () {
     var USER_0 = {
         userId: uuid.v4(),
         username: 'someusername',
-        password: 'somepassword',
+        password: 'Strong#$%2345',
         email: 'some@email.com',
         admin: true,
         salt: 'somesalt',

src/routes/test/cloudron-test.js

@@ -18,14 +18,14 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 var server;
 function setup(done) {
     nock.cleanAll();
+    config._reset();
     config.set('version', '0.5.0');
-    config.set('fqdn', 'localhost');
     server.start(done);
 }
 
@@ -33,6 +33,8 @@ function cleanup(done) {
     database._clear(function (error) {
         expect(error).to.not.be.ok();
 
+        config._reset();
+
         server.stop(done);
     });
 }
@@ -68,7 +70,7 @@ describe('Cloudron', function () {
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: 'someuser', password: 'somepassword', email: 'admin@foo.bar' })
+                   .send({ username: 'someuser', password: 'strong#A3asdf', email: 'admin@foo.bar' })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(500);
                 expect(scope.isDone()).to.be.ok();
@@ -81,7 +83,7 @@ describe('Cloudron', function () {
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: '', password: 'somepassword', email: 'admin@foo.bar' })
+                   .send({ username: '', password: 'ADSFsdf$%436', email: 'admin@foo.bar' })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(400);
                 expect(scope.isDone()).to.be.ok();
@@ -107,7 +109,7 @@ describe('Cloudron', function () {
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: 'someuser', password: 'somepassword', email: '' })
+                   .send({ username: 'someuser', password: 'ADSF#asd546', email: '' })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(400);
                 expect(scope.isDone()).to.be.ok();
@@ -115,12 +117,12 @@ describe('Cloudron', function () {
             });
         });
 
-        it('fails due to empty name', function (done) {
+        it('fails due to wrong displayName type', function (done) {
             var scope = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: 'someuser', password: '', email: 'admin@foo.bar', name: '' })
+                   .send({ username: 'someuser', password: 'ADSF?#asd546', email: 'admin@foo.bar', displayName: 1234 })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(400);
                 expect(scope.isDone()).to.be.ok();
@@ -133,7 +135,7 @@ describe('Cloudron', function () {
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: 'someuser', password: 'somepassword', email: 'invalidemail' })
+                   .send({ username: 'someuser', password: 'ADSF#asd546', email: 'invalidemail' })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(400);
                 expect(scope.isDone()).to.be.ok();
@@ -147,7 +149,7 @@ describe('Cloudron', function () {
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: 'someuser', password: 'somepassword', email: 'admin@foo.bar', name: 'tester' })
+                   .send({ username: 'someuser', password: 'ADSF#asd546', email: 'admin@foo.bar', displayName: 'tester' })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(201);
                 expect(scope1.isDone()).to.be.ok();
@@ -161,7 +163,7 @@ describe('Cloudron', function () {
 
             superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                    .query({ setupToken: 'somesetuptoken' })
-                   .send({ username: 'someuser', password: 'somepassword', email: 'admin@foo.bar' })
+                   .send({ username: 'someuser', password: 'ADSF#asd546', email: 'admin@foo.bar' })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(409);
                 expect(scope.isDone()).to.be.ok();
@@ -258,181 +260,6 @@ describe('Cloudron', function () {
 
     });
 
-    describe('migrate', function () {
-        before(function (done) {
-            async.series([
-                setup,
-
-                function (callback) {
-                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
-                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
-
-                    config._reset();
-
-                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
-                           .query({ setupToken: 'somesetuptoken' })
-                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                           .end(function (error, result) {
-                        expect(result).to.be.ok();
-                        expect(scope1.isDone()).to.be.ok();
-                        expect(scope2.isDone()).to.be.ok();
-
-                        // stash token for further use
-                        token = result.body.token;
-
-                        callback();
-                    });
-                },
-
-                function setupBackupConfig(callback) {
-                    superagent.post(SERVER_URL + '/api/v1/settings/backup_config')
-                           .send({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
-                           .query({ access_token: token })
-                           .end(function (error, result) {
-                        expect(result.statusCode).to.equal(200);
-
-                        callback();
-                    });
-                }
-
-            ], done);
-        });
-
-        after(cleanup);
-
-        it('fails without token', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 'small', region: 'sfo'})
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(401);
-                done();
-            });
-        });
-
-        it('fails without password', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 'small', region: 'sfo'})
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('fails with missing size', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ region: 'sfo', password: PASSWORD })
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('fails with wrong size type', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 4, region: 'sfo', password: PASSWORD })
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('fails with missing region', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 'small', password: PASSWORD })
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('fails with wrong region type', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 'small', region: 4, password: PASSWORD })
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('fails when in wrong state', function (done) {
-            var scope2 = nock(config.apiServerOrigin())
-                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
-                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
-
-            var scope3 = nock(config.apiServerOrigin())
-                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
-                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
-                    })
-                    .reply(200, { id: 'someid' });
-
-            var scope1 = nock(config.apiServerOrigin())
-                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
-                    return body.size && body.region && body.restoreKey;
-                }).reply(409, {});
-
-            injectShellMock();
-
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 'small', region: 'sfo', password: PASSWORD })
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(202);
-
-                function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
-                        restoreShellMock();
-                        return done();
-                    }
-
-                    setTimeout(checkAppstoreServerCalled, 100);
-                }
-
-                checkAppstoreServerCalled();
-            });
-        });
-
-        it('succeeds', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
-                return body.size && body.region && body.restoreKey;
-            }).reply(202, {});
-
-            var scope2 = nock(config.apiServerOrigin())
-                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
-                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
-                    })
-                    .reply(200, { id: 'someid' });
-
-            var scope3 = nock(config.apiServerOrigin())
-                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
-                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
-
-            injectShellMock();
-
-            superagent.post(SERVER_URL + '/api/v1/cloudron/migrate')
-                   .send({ size: 'small', region: 'sfo', password: PASSWORD })
-                   .query({ access_token: token })
-                   .end(function (error, result) {
-                expect(result.statusCode).to.equal(202);
-
-                function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
-                        restoreShellMock();
-                        return done();
-                    }
-
-                    setTimeout(checkAppstoreServerCalled, 100);
-                }
-
-                checkAppstoreServerCalled();
-            });
-        });
-    });
-
     describe('feedback', function () {
         before(function (done) {
             async.series([
@@ -514,7 +341,7 @@ describe('Cloudron', function () {
 
         it('succeeds with app type', function (done) {
             superagent.post(SERVER_URL + '/api/v1/cloudron/feedback')
-                   .send({ type: 'app', subject: 'some subject', description: 'some description' })
+                   .send({ type: 'app_missing', subject: 'some subject', description: 'some description' })
                    .query({ access_token: token })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(201);

src/routes/test/developer-test.js

@@ -17,7 +17,7 @@ var async = require('async'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null; // authentication token
 
 var server;

src/routes/test/groups-test.js

@@ -0,0 +1,245 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var appdb = require('../../appdb.js'),
+    async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    groups = require('../../groups.js'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    settings = require('../../settings.js'),
+    tokendb = require('../../tokendb.js'),
+    nock = require('nock'),
+    userdb = require('../../userdb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1337', EMAIL_1 ='happy@me.com';
+var token, token_1 = null;
+
+var server;
+function setup(done) {
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        },
+        function (callback) {
+            superagent.post(SERVER_URL + '/api/v1/users')
+                   .query({ access_token: token })
+                   .send({ username: USERNAME_1, email: EMAIL_1, invite: false })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+
+                token_1 = tokendb.generateToken();
+
+                // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
+            });
+        }
+  ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Groups API', function () {
+    before(setup);
+    after(cleanup);
+
+    describe('list', function () {
+        it('cannot get groups without token', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups')
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('cannot get groups as normal user', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups')
+                   .query({ access_token: token_1 })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(403);
+                done();
+            });
+        });
+
+        it('can get groups', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups')
+                   .query({ access_token: token })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.groups).to.be.an(Array);
+                expect(res.body.groups.length).to.be(1);
+                expect(res.body.groups[0].name).to.eql('admin');
+                done();
+            });
+        });
+    });
+
+    describe('create', function () {
+        it('fails due to mising token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/groups')
+                  .send({ name: 'externals'})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/groups')
+                  .query({ access_token: token })
+                  .send({ name: 'externals'})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(201);
+                done();
+            });
+        });
+
+        it('fails for already exists', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/groups')
+                  .query({ access_token: token })
+                  .send({ name: 'externals'})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+        });
+    });
+
+    describe('get', function () {
+        it('cannot get non-existing group', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/nope')
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        it('cannot get existing group with normal user', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/admin')
+                  .query({ access_token: token_1 })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(403);
+                done();
+            });
+        });
+
+        it('can get existing group', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/admin')
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.name).to.be('admin');
+                expect(result.body.userIds.length).to.be(1);
+                expect(result.body.userIds[0]).to.be(USERNAME);
+                done();
+            });
+        });
+    });
+
+    describe('remove', function () {
+        it('cannot remove without token', function (done) {
+            superagent.del(SERVER_URL + '/api/v1/groups/externals')
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('can remove empty group', function (done) {
+            superagent.del(SERVER_URL + '/api/v1/groups/externals')
+                  .send({ password: PASSWORD })
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('cannot remove non-empty group', function (done) {
+            superagent.del(SERVER_URL + '/api/v1/groups/admin')
+                  .send({ password: PASSWORD })
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(409);
+                done();
+            });
+        });
+    });
+
+    describe('Set groups', function () {
+        before(function (done) {
+            async.series([
+                groups.create.bind(null, 'group0'),
+                groups.create.bind(null, 'group1')
+            ], done);
+        });
+
+        it('cannot add user to invalid group', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'admin', 'something' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        it('can add user to valid group', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'admin', 'group0', 'group1' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('can remove last user from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME + '/set_groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'group0', 'group1' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(403); // not allowed
+                done();
+            });
+        });
+    });
+});

src/routes/test/oauth2-test.js

@@ -139,13 +139,13 @@ describe('OAuth2', function () {
         var USER_0 = {
             id: uuid.v4(),
             username: 'someusername',
-            password: 'somepassword',
+            password: '@#45Strongpassword',
             email: 'some@email.com',
-            admin: true,
             salt: 'somesalt',
             createdAt: (new Date()).toUTCString(),
             modifiedAt: (new Date()).toUTCString(),
-            resetToken: hat(256)
+            resetToken: hat(256),
+            displayName: ''
         };
 
         var APP_0 = {
@@ -155,7 +155,7 @@ describe('OAuth2', function () {
             location: 'test',
             portBindings: {},
             accessRestriction: null,
-            oauthProxy: true
+            memoryLimit: 0
         };
 
         var APP_1 = {
@@ -165,7 +165,7 @@ describe('OAuth2', function () {
             location: 'test1',
             portBindings: {},
             accessRestriction: { users: [ 'foobar' ] },
-            oauthProxy: true
+            memoryLimit: 0
         };
 
         var APP_2 = {
@@ -175,7 +175,17 @@ describe('OAuth2', function () {
             location: 'test2',
             portBindings: {},
             accessRestriction: { users: [ USER_0.id ] },
-            oauthProxy: true
+            memoryLimit: 0
+        };
+
+        var APP_3 = {
+            id: 'app3',
+            appStoreId: '',
+            manifest: { version: '0.1.0', addons: { } },
+            location: 'test3',
+            portBindings: {},
+            accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
+            memoryLimit: 0
         };
 
         // unknown app
@@ -268,6 +278,16 @@ describe('OAuth2', function () {
             scope: 'profile'
         };
 
+        // app with accessRestriction allowing group
+        var CLIENT_9 = {
+            id: 'cid-client9',
+            appId: APP_3.id,
+            type: clientdb.TYPE_OAUTH,
+            clientSecret: 'secret9',
+            redirectURI: 'http://redirect9',
+            scope: 'profile'
+        };
+
         // make csrf always succeed for testing
         oauth2.csrf = function (req, res, next) {
             req.csrfToken = function () { return hat(256); };
@@ -287,11 +307,13 @@ describe('OAuth2', function () {
                 clientdb.add.bind(null, CLIENT_6.id, CLIENT_6.appId, CLIENT_6.type, CLIENT_6.clientSecret, CLIENT_6.redirectURI, CLIENT_6.scope),
                 clientdb.add.bind(null, CLIENT_7.id, CLIENT_7.appId, CLIENT_7.type, CLIENT_7.clientSecret, CLIENT_7.redirectURI, CLIENT_7.scope),
                 clientdb.add.bind(null, CLIENT_8.id, CLIENT_8.appId, CLIENT_8.type, CLIENT_8.clientSecret, CLIENT_8.redirectURI, CLIENT_8.scope),
-                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.oauthProxy),
-                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.oauthProxy),
-                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.oauthProxy),
+                clientdb.add.bind(null, CLIENT_9.id, CLIENT_9.appId, CLIENT_9.type, CLIENT_9.clientSecret, CLIENT_9.redirectURI, CLIENT_9.scope),
+                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
+                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
+                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit),
+                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit),
                 function (callback) {
-                    user.create(USER_0.username, USER_0.password, USER_0.email, true, '', function (error, userObject) {
+                    user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, function (error, userObject) {
                         expect(error).to.not.be.ok();
 
                         // update the global objects to reflect the new user id
@@ -777,7 +799,7 @@ describe('OAuth2', function () {
                         expect(error).to.not.be.ok();
                         expect(response.statusCode).to.eql(200);
                         expect(body.indexOf('<!-- error tester -->')).to.not.equal(-1);
-                        expect(body.indexOf('Unkonwn OAuth client.')).to.not.equal(-1);
+                        expect(body.indexOf('Unknown OAuth client.')).to.not.equal(-1);
 
                         done();
                     });
@@ -801,6 +823,21 @@ describe('OAuth2', function () {
                 });
             });
 
+            it('fails for grant type code with accessRestriction (group)', function (done) { // USER_0 is not an admin
+                startAuthorizationFlow(CLIENT_9, 'code', function (jar) {
+                    var url = SERVER_URL + '/api/v1/oauth/dialog/authorize?redirect_uri=' + CLIENT_9.redirectURI + '&client_id=' + CLIENT_9.id + '&response_type=code';
+
+                    request.get(url, { jar: jar, followRedirect: false }, function (error, response, body) {
+                        expect(error).to.not.be.ok();
+                        expect(response.statusCode).to.eql(200);
+                        expect(body.indexOf('<!-- error tester -->')).to.not.equal(-1);
+                        expect(body.indexOf('No access to this app.')).to.not.equal(-1);
+
+                        done();
+                    });
+                });
+            });
+
             it('fails for grant type token due to accessRestriction', function (done) {
                 startAuthorizationFlow(CLIENT_6, 'token', function (jar) {
                     var url = SERVER_URL + '/api/v1/oauth/dialog/authorize?redirect_uri=' + CLIENT_6.redirectURI + '&client_id=' + CLIENT_6.id + '&response_type=token';
@@ -824,7 +861,7 @@ describe('OAuth2', function () {
                         expect(error).to.not.be.ok();
                         expect(response.statusCode).to.eql(200);
                         expect(body.indexOf('<!-- error tester -->')).to.not.equal(-1);
-                        expect(body.indexOf('Unkonwn OAuth client.')).to.not.equal(-1);
+                        expect(body.indexOf('Unknown OAuth client.')).to.not.equal(-1);
 
                         done();
                     });
@@ -1239,13 +1276,14 @@ describe('Password', function () {
     var USER_0 = {
         userId: uuid.v4(),
         username: 'someusername',
-        password: 'somepassword',
+        password: 'passWord%1234',
         email: 'some@email.com',
         admin: true,
         salt: 'somesalt',
         createdAt: (new Date()).toUTCString(),
         modifiedAt: (new Date()).toUTCString(),
-        resetToken: hat(256)
+        resetToken: hat(256),
+        displayName: ''
     };
 
     // make csrf always succeed for testing
@@ -1405,6 +1443,15 @@ describe('Password', function () {
             });
         });
 
+        it('fails due to weak password', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/session/password/reset')
+            .send({ password: 'foobar', resetToken: USER_0.resetToken })
+            .end(function (error, result) {
+                expect(result.statusCode).to.equal(406);
+                done();
+            });
+        });
+
         it('succeeds', function (done) {
             var scope = nock(config.adminOrigin())
                 .filteringPath(function (path) {
@@ -1415,7 +1462,7 @@ describe('Password', function () {
                 .get('/?accessToken=token&expiresAt=1234').reply(200, {});
 
             superagent.post(SERVER_URL + '/api/v1/session/password/reset')
-            .send({ password: 'somepassword', resetToken: USER_0.resetToken })
+            .send({ password: 'ASF23$%somepassword', resetToken: USER_0.resetToken })
             .end(function (error, result) {
                 expect(scope.isDone()).to.be.ok();
                 expect(result.statusCode).to.equal(200);

src/routes/test/settings-test.js

@@ -22,7 +22,7 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 var token = null;
 
 var server;
@@ -56,7 +56,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, false /* oauthProxy */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, callback);
         }
     ], done);
 }

src/routes/test/simpleauth-test.js

@@ -21,7 +21,7 @@ describe('SimpleAuth API', function () {
     var SERVER_URL = 'http://localhost:' + config.get('port');
     var SIMPLE_AUTH_ORIGIN = 'http://localhost:' + config.get('simpleAuthPort');
 
-    var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+    var USERNAME = 'admin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
 
     var APP_0 = {
         id: 'app0',
@@ -30,7 +30,7 @@ describe('SimpleAuth API', function () {
         location: 'test0',
         portBindings: {},
         accessRestriction: { users: [ 'foobar', 'someone'] },
-        oauthProxy: true
+        memoryLimit: 0
     };
 
     var APP_1 = {
@@ -40,7 +40,7 @@ describe('SimpleAuth API', function () {
         location: 'test1',
         portBindings: {},
         accessRestriction: { users: [ 'foobar', USERNAME, 'someone' ] },
-        oauthProxy: true
+        memoryLimit: 0
     };
 
     var APP_2 = {
@@ -50,7 +50,17 @@ describe('SimpleAuth API', function () {
         location: 'test2',
         portBindings: {},
         accessRestriction: null,
-        oauthProxy: true
+        memoryLimit: 0
+    };
+
+    var APP_3 = {
+        id: 'app3',
+        appStoreId: '',
+        manifest: { version: '0.1.0', addons: { } },
+        location: 'test3',
+        portBindings: {},
+        accessRestriction: { groups: [ 'someothergroup', 'admin', 'anothergroup' ] },
+        memoryLimit: 0
     };
 
     var CLIENT_0 = {
@@ -98,6 +108,15 @@ describe('SimpleAuth API', function () {
         scope: 'user,profile'
     };
 
+    var CLIENT_5 = {
+        id: 'someclientid5',
+        appId: APP_3.id,
+        type: clientdb.TYPE_SIMPLE_AUTH,
+        clientSecret: 'someclientsecret5',
+        redirectURI: '',
+        scope: 'user,profile'
+    };
+
     before(function (done) {
         async.series([
             server.start.bind(server),
@@ -128,9 +147,11 @@ describe('SimpleAuth API', function () {
             clientdb.add.bind(null, CLIENT_2.id, CLIENT_2.appId, CLIENT_2.type, CLIENT_2.clientSecret, CLIENT_2.redirectURI, CLIENT_2.scope),
             clientdb.add.bind(null, CLIENT_3.id, CLIENT_3.appId, CLIENT_3.type, CLIENT_3.clientSecret, CLIENT_3.redirectURI, CLIENT_3.scope),
             clientdb.add.bind(null, CLIENT_4.id, CLIENT_4.appId, CLIENT_4.type, CLIENT_4.clientSecret, CLIENT_4.redirectURI, CLIENT_4.scope),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.oauthProxy),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.oauthProxy),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.oauthProxy)
+            clientdb.add.bind(null, CLIENT_5.id, CLIENT_5.appId, CLIENT_5.type, CLIENT_5.clientSecret, CLIENT_5.redirectURI, CLIENT_5.scope),
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit),
+            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit)
         ], done);
     });
 
@@ -333,6 +354,37 @@ describe('SimpleAuth API', function () {
             });
         });
 
+        it('succeeds for app with group accessRestriction', function (done) {
+            var body = {
+                clientId: CLIENT_5.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            superagent.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.accessToken).to.be.a('string');
+                expect(result.body.user).to.be.an('object');
+                expect(result.body.user.id).to.be.a('string');
+                expect(result.body.user.username).to.be.a('string');
+                expect(result.body.user.email).to.be.a('string');
+                expect(result.body.user.admin).to.be.a('boolean');
+
+                superagent.get(SERVER_URL + '/api/v1/profile')
+                .query({ access_token: result.body.accessToken })
+                .end(function (error, result) {
+                    expect(error).to.be(null);
+                    expect(result.body).to.be.an('object');
+                    expect(result.body.username).to.eql(USERNAME);
+
+                    done();
+                });
+            });
+        });
+
         it('fails for wrong client credentials', function (done) {
             var body = {
                 clientId: CLIENT_4.id,

src/routes/test/start_addons.sh

@@ -79,7 +79,7 @@ start_mongodb
 start_mail
 
 echo -n "Waiting for addons to start"
-for i in {1..20}; do
+for i in {1..10}; do
     echo -n "."
     sleep 1
 done

src/routes/test/user-test.js

@@ -10,6 +10,8 @@ var config = require('../../config.js'),
     database = require('../../database.js'),
     tokendb = require('../../tokendb.js'),
     expect = require('expect.js'),
+    groups = require('../../groups.js'),
+    mailer = require('../../mailer.js'),
     superagent = require('superagent'),
     nock = require('nock'),
     server = require('../../server.js'),
@@ -17,16 +19,23 @@ var config = require('../../config.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
-var USERNAME_0 = 'admin', PASSWORD = 'password', EMAIL = 'silly@me.com', EMAIL_0_NEW = 'stupid@me.com';
+var USERNAME_0 = 'admin', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silly@me.com', EMAIL_0_NEW = 'stupid@me.com', DISPLAY_NAME_0_NEW = 'New Name';
 var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'tao@zen.mac';
-var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'user@foo.bar';
+var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'user@foo.bar', EMAIL_2_NEW = 'happy@me.com';
 var USERNAME_3 = 'userTheThird', EMAIL_3 = 'user3@foo.bar';
 
 var server;
 function setup(done) {
     server.start(function (error) {
         expect(!error).to.be.ok();
-        userdb._clear(done);
+
+        mailer._clearMailQueue();
+
+        userdb._clear(function (error) {
+            expect(error).to.eql(null);
+
+            groups.create('somegroupid', done);
+        });
     });
 }
 
@@ -34,10 +43,21 @@ function cleanup(done) {
     database._clear(function (error) {
         expect(!error).to.be.ok();
 
+        mailer._clearMailQueue();
+
         server.stop(done);
     });
 }
 
+function checkMails(number, done) {
+    // mails are enqueued async
+    setTimeout(function () {
+        expect(mailer._getMailQueue().length).to.equal(number);
+        mailer._clearMailQueue();
+        done();
+    }, 500);
+}
+
 describe('User API', function () {
     this.timeout(5000);
 
@@ -85,7 +105,7 @@ describe('User API', function () {
 
         superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
                .query({ setupToken: 'somesetuptoken' })
-               .send({ username: USERNAME_0, password: PASSWORD, email: EMAIL })
+               .send({ username: USERNAME_0, password: PASSWORD, email: EMAIL_0 })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(201);
 
@@ -113,7 +133,7 @@ describe('User API', function () {
         .end(function (err, res) {
             expect(res.statusCode).to.equal(200);
             expect(res.body.username).to.equal(USERNAME_0);
-            expect(res.body.email).to.equal(EMAIL);
+            expect(res.body.email).to.equal(EMAIL_0);
             expect(res.body.admin).to.be.ok();
 
             // stash for further use
@@ -147,7 +167,7 @@ describe('User API', function () {
                .end(function (err, res) {
             expect(res.statusCode).to.equal(200);
             expect(res.body.username).to.equal(USERNAME_0);
-            expect(res.body.email).to.equal(EMAIL);
+            expect(res.body.email).to.equal(EMAIL_0);
             expect(res.body.admin).to.be.ok();
             done();
         });
@@ -186,8 +206,9 @@ describe('User API', function () {
                .end(function (err, res) {
             expect(res.statusCode).to.equal(200);
             expect(res.body.username).to.equal(USERNAME_0);
-            expect(res.body.email).to.equal(EMAIL);
+            expect(res.body.email).to.equal(EMAIL_0);
             expect(res.body.admin).to.be.ok();
+            expect(res.body.displayName).to.be.a('string');
             expect(res.body.password).to.not.be.ok();
             expect(res.body.salt).to.not.be.ok();
             done();
@@ -213,66 +234,91 @@ describe('User API', function () {
     });
 
     it('create second user succeeds', function (done) {
+        mailer._clearMailQueue();
+
         superagent.post(SERVER_URL + '/api/v1/users')
                .query({ access_token: token })
-               .send({ username: USERNAME_1, email: EMAIL_1 })
+               .send({ username: USERNAME_1, email: EMAIL_1, invite: true })
                .end(function (err, res) {
             expect(err).to.not.be.ok();
             expect(res.statusCode).to.equal(201);
 
-            // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-            tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 10000, '*', done);
+            checkMails(2, function () {
+              // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+              tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 10000, '*', done);
+            });
+        });
+    });
+
+    it('reinvite unknown user fails', function (done) {
+        mailer._clearMailQueue();
+
+        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_1+USERNAME_1 + '/invite')
+               .query({ access_token: token })
+               .send({})
+               .end(function (err, res) {
+            expect(err).to.be.an(Error);
+            expect(res.statusCode).to.equal(404);
+            checkMails(0, done);
+        });
+    });
+
+    it('reinvite second user succeeds', function (done) {
+        mailer._clearMailQueue();
+
+        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/invite')
+               .query({ access_token: token })
+               .send({})
+               .end(function (err, res) {
+            expect(err).to.not.be.ok();
+            expect(res.statusCode).to.equal(200);
+            checkMails(1, done);
         });
     });
 
     it('set second user as admin succeeds', function (done) {
-        // TODO is USERNAME_1 in body and url redundant?
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/admin')
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/set_groups')
                .query({ access_token: token })
-               .send({ username: USERNAME_1, admin: true })
+               .send({ groupIds: [ groups.ADMIN_GROUP_ID ] })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(204);
-            done();
+
+            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+               .query({ access_token: token })
+               .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.admin).to.equal(true);
+
+                done();
+            });
         });
     });
 
-    it('remove first user from admins succeeds', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/admin')
-               .query({ access_token: token_1 })
-               .send({ username: USERNAME_0, admin: false })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(204);
-            done();
-        });
-    });
-
-    it('remove second user by first, now normal, user fails', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+    it('remove itself from admins fails', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/set_groups')
                .query({ access_token: token })
-               .send({ password: PASSWORD })
+               .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(403);
             done();
         });
     });
 
-    it('remove second user from admins and thus last admin fails', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/admin')
-               .query({ access_token: token_1 })
-               .send({ username: USERNAME_1, admin: false })
-               .end(function (err, res) {
-            expect(res.statusCode).to.equal(403);
-            done();
-        });
-    });
-
-    it('reset first user as admin succeeds', function (done) {
-        superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/admin')
-               .query({ access_token: token_1 })
-               .send({ username: USERNAME_0, admin: true })
+    it('remove second user from admins succeeds', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_1 + '/set_groups')
+               .query({ access_token: token })
+               .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(204);
-            done();
+
+            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_1)
+               .query({ access_token: token })
+               .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.admin).to.equal(false);
+
+                done();
+            });
         });
     });
 
@@ -296,29 +342,53 @@ describe('User API', function () {
         });
     });
 
-    it('create second and third user', function (done) {
+    it('create user missing invite fails', function (done) {
         superagent.post(SERVER_URL + '/api/v1/users')
                .query({ access_token: token })
                .send({ username: USERNAME_2, email: EMAIL_2 })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('create second and third user', function (done) {
+        mailer._clearMailQueue();
+
+        superagent.post(SERVER_URL + '/api/v1/users')
+               .query({ access_token: token })
+               .send({ username: USERNAME_2, email: EMAIL_2, invite: false })
                .end(function (error, res) {
             expect(res.statusCode).to.equal(201);
 
             superagent.post(SERVER_URL + '/api/v1/users')
                    .query({ access_token: token })
-                   .send({ username: USERNAME_3, email: EMAIL_3 })
+                   .send({ username: USERNAME_3, email: EMAIL_3, invite: true })
                    .end(function (error, res) {
                 expect(res.statusCode).to.equal(201);
 
-                // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_2, tokendb.PREFIX_USER + USERNAME_2, 'test-client-id',  Date.now() + 10000, '*', done);
+                // one mail for first user creation, two mails for second user creation (see 'invite' flag)
+                checkMails(3, function () {
+                    // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
+                    tokendb.add(token_2, tokendb.PREFIX_USER + USERNAME_2, 'test-client-id',  Date.now() + 10000, '*', done);
+                });
             });
         });
     });
 
-    it('second user userInfo', function (done) {
+    it('second user userInfo fails for first user', function (done) {
         superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
                .query({ access_token: token_1 })
                .end(function (error, result) {
+            expect(result.statusCode).to.equal(403);
+            done();
+        });
+    });
+
+    it('second user userInfo succeeds for second user', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+               .query({ access_token: token_2 })
+               .end(function (error, result) {
             expect(result.statusCode).to.equal(200);
             expect(result.body.username).to.equal(USERNAME_2);
             expect(result.body.email).to.equal(EMAIL_2);
@@ -331,16 +401,25 @@ describe('User API', function () {
     it('create user with same username should fail', function (done) {
         superagent.post(SERVER_URL + '/api/v1/users')
                .query({ access_token: token })
-               .send({ username: USERNAME_2, email: EMAIL })
+               .send({ username: USERNAME_2, email: EMAIL_0, invite: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(409);
             done();
         });
     });
 
-    it('list users', function (done) {
+    it('list users fails for normal user', function (done) {
         superagent.get(SERVER_URL + '/api/v1/users')
         .query({ access_token: token_2 })
+        .end(function (error, res) {
+            expect(res.statusCode).to.equal(403);
+            done();
+        });
+    });
+
+    it('list users succeeds for admin', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/users')
+        .query({ access_token: token })
         .end(function (error, res) {
             expect(error).to.be(null);
             expect(res.statusCode).to.equal(200);
@@ -371,7 +450,7 @@ describe('User API', function () {
     });
 
     it('admin cannot remove normal user without giving a password', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_3)
+        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
                .query({ access_token: token })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
@@ -380,7 +459,7 @@ describe('User API', function () {
     });
 
     it('admin cannot remove normal user with empty password', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_3)
+        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
                .query({ access_token: token })
                .send({ password: '' })
                .end(function (err, res) {
@@ -390,7 +469,7 @@ describe('User API', function () {
     });
 
     it('admin cannot remove normal user with giving wrong password', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_3)
+        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
                .query({ access_token: token })
                .send({ password: PASSWORD + PASSWORD })
                .end(function (err, res) {
@@ -400,7 +479,7 @@ describe('User API', function () {
     });
 
     it('admin removes normal user', function (done) {
-        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_3)
+        superagent.del(SERVER_URL + '/api/v1/users/' + USERNAME_1)
                .query({ access_token: token })
                .send({ password: PASSWORD })
                .end(function (err, res) {
@@ -422,29 +501,9 @@ describe('User API', function () {
     // Change email
     it('change email fails due to missing token', function (done) {
         superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
-               .send({ password: PASSWORD, email: EMAIL_0_NEW })
-               .end(function (error, result) {
-            expect(result.statusCode).to.equal(401);
-            done();
-        });
-    });
-
-    it('change email fails due to missing password', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
-               .query({ access_token: token })
                .send({ email: EMAIL_0_NEW })
                .end(function (error, result) {
-            expect(result.statusCode).to.equal(400);
-            done();
-        });
-    });
-
-    it('change email fails due to wrong password', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
-               .query({ access_token: token })
-               .send({ password: PASSWORD+PASSWORD, email: EMAIL_0_NEW })
-               .end(function (error, result) {
-            expect(result.statusCode).to.equal(403);
+            expect(result.statusCode).to.equal(401);
             done();
         });
     });
@@ -452,20 +511,93 @@ describe('User API', function () {
     it('change email fails due to invalid email', function (done) {
         superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
                .query({ access_token: token })
-               .send({ password: PASSWORD, email: 'foo@bar' })
+               .send({ email: 'foo@bar' })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(400);
             done();
         });
     });
 
-    it('change email succeeds', function (done) {
+    it('change email for other user fails', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+               .query({ access_token: token_2 })
+               .send({ email: 'foobar@bar.baz' })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(403);
+            done();
+        });
+    });
+
+    it('change user succeeds without email nor displayName', function (done) {
         superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
                .query({ access_token: token })
-               .send({ password: PASSWORD, email: EMAIL_0_NEW })
+               .send({})
                .end(function (error, result) {
             expect(result.statusCode).to.equal(204);
-            done(error);
+            done();
+        });
+    });
+
+    it('change email for own user succeeds', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+               .query({ access_token: token_2 })
+               .send({ email: EMAIL_2_NEW })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(204);
+
+            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+                  .query({ access_token: token_2 })
+                  .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.username).to.equal(USERNAME_2);
+                expect(res.body.email).to.equal(EMAIL_2_NEW);
+                expect(res.body.admin).to.equal(false);
+                expect(res.body.displayName).to.equal('');
+
+                done();
+            });
+        });
+    });
+
+    it('change email as admin for other user succeeds', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+               .query({ access_token: token })
+               .send({ email: EMAIL_2 })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(204);
+
+            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_2)
+                  .query({ access_token: token })
+                  .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.username).to.equal(USERNAME_2);
+                expect(res.body.email).to.equal(EMAIL_2);
+                expect(res.body.admin).to.equal(false);
+                expect(res.body.displayName).to.equal('');
+
+                done();
+            });
+        });
+    });
+
+    it('change displayName succeeds', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+               .query({ access_token: token })
+               .send({ displayName: DISPLAY_NAME_0_NEW })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(204);
+
+            superagent.get(SERVER_URL + '/api/v1/users/' + USERNAME_0)
+                  .query({ access_token: token })
+                  .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.username).to.equal(USERNAME_0);
+                expect(res.body.email).to.equal(EMAIL_0);
+                expect(res.body.admin).to.be.ok();
+                expect(res.body.displayName).to.equal(DISPLAY_NAME_0_NEW);
+
+                done();
+            });
         });
     });
 
@@ -493,7 +625,7 @@ describe('User API', function () {
     it('change password fails due to wrong password', function (done) {
         superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
                .query({ access_token: token })
-               .send({ password: 'some wrong password', newPassword: 'newpassword' })
+               .send({ password: 'some wrong password', newPassword: 'MOre#$%34' })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(403);
             done();
@@ -513,7 +645,7 @@ describe('User API', function () {
     it('change password succeeds', function (done) {
         superagent.post(SERVER_URL + '/api/v1/users/' + USERNAME_0 + '/password')
                .query({ access_token: token })
-               .send({ password: PASSWORD, newPassword: 'new_password' })
+               .send({ password: PASSWORD, newPassword: 'MOre#$%34' })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(204);
             done();

src/routes/user.js

@@ -9,14 +9,16 @@ exports = module.exports = {
     list: listUser,
     create: createUser,
     changePassword: changePassword,
-    changeAdmin: changeAdmin,
     remove: removeUser,
     verifyPassword: verifyPassword,
-    requireAdmin: requireAdmin
+    requireAdmin: requireAdmin,
+    sendInvite: sendInvite,
+    setGroups: setGroups
 };
 
 var assert = require('assert'),
-    generatePassword = require('password-generator'),
+    generatePassword = require('../password.js').generate,
+    groups = require('../groups.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     user = require('../user.js'),
@@ -33,10 +35,18 @@ function profile(req, res, next) {
     if (req.user.tokenType === tokendb.TYPE_USER || req.user.tokenType === tokendb.TYPE_DEV) {
         result.username = req.user.username;
         result.email = req.user.email;
-        result.admin = req.user.admin;
-    }
+        result.displayName = req.user.displayName;
 
-    next(new HttpSuccess(200, result));
+        groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
+            if (error) return next(new HttpError(500, error));
+
+            result.admin = isAdmin;
+
+            next(new HttpSuccess(200, result));
+        });
+    } else {
+        next(new HttpSuccess(200, result));
+    }
 }
 
 function createUser(req, res, next) {
@@ -44,12 +54,16 @@ function createUser(req, res, next) {
 
     if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be string'));
     if (typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if (typeof req.body.invite !== 'boolean') return next(new HttpError(400, 'invite must be boolean'));
+    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
 
     var username = req.body.username;
-    var password = generatePassword(8, true /* memorable */);
+    var password = generatePassword();
     var email = req.body.email;
+    var sendInvite = req.body.invite;
+    var displayName = req.body.displayName || '';
 
-    user.create(username, password, email, false /* admin */, req.user /* creator */, function (error, user) {
+    user.create(username, password, email, displayName, { invitor: req.user, sendInvite: sendInvite }, function (error, user) {
         if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, 'Invalid username'));
         if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, 'Invalid email'));
         if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, 'Invalid password'));
@@ -70,33 +84,27 @@ function createUser(req, res, next) {
 }
 
 function update(req, res, next) {
+    assert.strictEqual(typeof req.params.userId, 'string');
     assert.strictEqual(typeof req.user, 'object');
     assert.strictEqual(typeof req.body, 'object');
 
-    if (typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
+    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
 
     if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+    if (req.user.id !== req.params.userId && !req.user.admin) return next(new HttpError(403, 'Not allowed'));
 
-    user.update(req.user.id, req.user.username, req.body.email, function (error) {
-        if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+    user.get(req.params.userId, function (error, result) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(204));
-    });
-}
+        user.update(req.params.userId, result.username, req.body.email || result.email, req.body.displayName || result.displayName, function (error) {
+            if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
+            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+            if (error) return next(new HttpError(500, error));
 
-function changeAdmin(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'API call requires a username.'));
-    if (typeof req.body.admin !== 'boolean') return next(new HttpError(400, 'API call requires an admin setting.'));
-
-    user.changeAdmin(req.body.username, req.body.admin, function (error) {
-        if (error && error.reason === UserError.NOT_ALLOWED) return next(new HttpError(403, 'Last admin'));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(204));
+            next(new HttpSuccess(204));
+        });
     });
 }
 
@@ -128,17 +136,25 @@ function listUser(req, res, next) {
 
 function info(req, res, next) {
     assert.strictEqual(typeof req.params.userId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (req.user.id !== req.params.userId && !req.user.admin) return next(new HttpError(403, 'Not allowed'));
 
     user.get(req.params.userId, function (error, result) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(200, {
-            id: result.id,
-            username: result.username,
-            email: result.email,
-            admin: result.admin
-        }));
+        groups.isMember(groups.ADMIN_GROUP_ID, req.params.userId, function (error, isAdmin) {
+            if (error) return next(new HttpError(500, error));
+
+            next(new HttpSuccess(200, {
+                id: result.id,
+                username: result.username,
+                email: result.email,
+                admin: isAdmin,
+                displayName: result.displayName
+            }));
+        });
     });
 }
 
@@ -168,12 +184,19 @@ function verifyPassword(req, res, next) {
 
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires user password'));
 
-    user.verify(req.user.username, req.body.password, function (error) {
-        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new HttpError(403, 'Password incorrect'));
-        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Password incorrect'));
+    groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
         if (error) return next(new HttpError(500, error));
 
-        next();
+        // Only allow admins or users, operating on themselves
+        if (req.params.userId && !(req.user.id === req.params.userId || isAdmin)) return next(new HttpError(403, 'Not allowed'));
+
+        user.verify(req.user.username, req.body.password, function (error) {
+            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new HttpError(403, 'Password incorrect'));
+            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Password incorrect'));
+            if (error) return next(new HttpError(500, error));
+
+            next();
+        });
     });
 }
 
@@ -188,3 +211,30 @@ function requireAdmin(req, res, next) {
     next();
 }
 
+function sendInvite(req, res, next) {
+    assert.strictEqual(typeof req.params.userId, 'string');
+
+    user.sendInvite(req.params.userId, function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function setGroups(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.userId, 'string');
+
+    if (!Array.isArray(req.body.groupIds)) return next(new HttpError(400, 'API call requires a groups array.'));
+
+    // this route is only allowed for admins, so req.user has to be an admin
+    if (req.user.id === req.params.userId && req.body.groupIds.indexOf(groups.ADMIN_GROUP_ID) === -1) return next(new HttpError(403, 'Admin removing itself from admins is not allowed'));
+
+    user.setGroups(req.params.userId, req.body.groupIds, function (error) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'One or more groups not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/scripts/backupapp.sh

@@ -12,8 +12,8 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [ $# -lt 3 ]; then
-    echo "Usage: backupapp.sh <appid> <url> <key> [aws session token]"
+if [ $# -lt 4 ]; then
+    echo "Usage: backupapp.sh <appid> <url> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -21,8 +21,9 @@ readonly DATA_DIR="${HOME}/data"
 
 app_id="$1"
 backup_url="$2"
-backup_key="$3"
-session_token="$4"
+backup_config_url="$3"
+backup_key="$4"
+session_token="$5"
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
@@ -48,12 +49,34 @@ for try in `seq 1 5`; do
     cat "${error_log}" && rm "${error_log}"
 done
 
+if [[ ${try} -eq 5 ]]; then
+    echo "Backup failed uploading backup tarball"
+    exit 1
+fi
+
+for try in `seq 1 5`; do
+    echo "Uploading config.json to ${backup_config_url} (try ${try})"
+    error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
+    if cat "${app_data_snapshot}/config.json" \
+           | curl --fail -X PUT ${headers[@]} --data @- "${backup_config_url}" 2>"${error_log}"; then
+        break
+    fi
+    cat "${error_log}" && rm "${error_log}"
+done
+
 btrfs subvolume delete "${app_data_snapshot}"
 
 if [[ ${try} -eq 5 ]]; then
-    echo "Backup failed"
+    echo "Backup failed uploading config.json"
     exit 1
 else
     echo "Backup successful"
 fi
-

src/scripts/retire.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# This script is called once at the end of a cloudrons lifetime
+
+set -eu -o pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+readonly BOX_SRC_DIR=/home/yellowtent/box
+
+if [[ $# == 1 && "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+echo "Retiring cloudron"
+
+if [[ "${BOX_ENV}" != "cloudron" ]]; then
+	exit 0
+fi
+
+"${BOX_SRC_DIR}/setup/splashpage.sh" --retire --data "$1" # show splash
+
+echo "Stopping apps"
+systemctl stop docker # stop the apps
+
+echo "Stopping installer"
+systemctl stop cloudron-installer # stop the installer
+
+# do this at the end since stopping the box will kill this script as well
+echo "Stopping Cloudron Smartserver"
+"${BOX_SRC_DIR}/setup/stop.sh"

src/server.js

@@ -34,13 +34,13 @@ function initializeExpressSync() {
     var QUERY_LIMIT = '10mb', // max size for json and urlencoded queries
         FIELD_LIMIT = 2 * 1024 * 1024; // max fields that can appear in multipart
 
-    var REQUEST_TIMEOUT = 10000; // timeout for all requests
+    var REQUEST_TIMEOUT = 10000; // timeout for all requests (see also setTimeout on the httpServer)
 
     var json = middleware.json({ strict: true, limit: QUERY_LIMIT }), // application/json
         urlencoded = middleware.urlencoded({ extended: false, limit: QUERY_LIMIT }); // application/x-www-form-urlencoded
 
     app.set('views', path.join(__dirname, 'oauth2views'));
-    app.set('view options', { layout: true, debug: true });
+    app.set('view options', { layout: true, debug: false });
     app.set('view engine', 'ejs');
 
     if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('Box :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
@@ -93,7 +93,6 @@ function initializeExpressSync() {
     router.get ('/api/v1/cloudron/config', rootScope, routes.cloudron.getConfig);
     router.post('/api/v1/cloudron/update', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
     router.post('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
-    router.post('/api/v1/cloudron/migrate', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.migrate);
     router.get ('/api/v1/cloudron/graphs', rootScope, routes.graphs.getGraphs);
 
     // feedback
@@ -101,13 +100,23 @@ function initializeExpressSync() {
 
     router.get ('/api/v1/profile', profileScope, routes.user.profile);
 
-    router.get ('/api/v1/users', usersScope, routes.user.list);
+    // user routes only for admins
+    router.get ('/api/v1/users', usersScope, routes.user.requireAdmin, routes.user.list);
     router.post('/api/v1/users', usersScope, routes.user.requireAdmin, routes.user.create);
-    router.get ('/api/v1/users/:userId', usersScope, routes.user.info);
-    router.put ('/api/v1/users/:userId', usersScope, routes.user.verifyPassword, routes.user.update);
     router.del ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.user.remove);
+    router.put ('/api/v1/users/:userId/set_groups', usersScope, routes.user.requireAdmin, routes.user.setGroups);
+    router.post('/api/v1/users/:userId/invite', usersScope, routes.user.requireAdmin, routes.user.sendInvite);
+
+    // user routes for admins and users operating on their own account
+    router.get ('/api/v1/users/:userId', usersScope, routes.user.info);
+    router.put ('/api/v1/users/:userId', usersScope, routes.user.update);
     router.post('/api/v1/users/:userId/password', usersScope, routes.user.changePassword); // changePassword verifies password
-    router.post('/api/v1/users/:userId/admin', usersScope, routes.user.requireAdmin, routes.user.changeAdmin);
+
+    // Group management
+    router.get ('/api/v1/groups', usersScope, routes.user.requireAdmin, routes.groups.list);
+    router.post('/api/v1/groups', usersScope, routes.user.requireAdmin, routes.groups.create);
+    router.get ('/api/v1/groups/:groupId', usersScope, routes.user.requireAdmin, routes.groups.get);
+    router.del ('/api/v1/groups/:groupId', usersScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.groups.remove);
 
     // form based login routes used by oauth2 frame
     router.get ('/api/v1/session/login', csrf, routes.oauth2.loginForm);
@@ -143,6 +152,7 @@ function initializeExpressSync() {
     router.post('/api/v1/apps/:id/update',    appsScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.apps.updateApp);
     router.post('/api/v1/apps/:id/restore',   appsScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.apps.restoreApp);
     router.post('/api/v1/apps/:id/backup',    appsScope, routes.user.requireAdmin, routes.apps.backupApp);
+    router.get ('/api/v1/apps/:id/backups',   appsScope, routes.user.requireAdmin, routes.apps.listBackups);
     router.post('/api/v1/apps/:id/stop',      appsScope, routes.user.requireAdmin, routes.apps.stopApp);
     router.post('/api/v1/apps/:id/start',     appsScope, routes.user.requireAdmin, routes.apps.startApp);
     router.get ('/api/v1/apps/:id/logstream', appsScope, routes.user.requireAdmin, routes.apps.getLogStream);
@@ -169,6 +179,10 @@ function initializeExpressSync() {
     // backup routes
     router.get ('/api/v1/backups', settingsScope, routes.backups.get);
     router.post('/api/v1/backups', settingsScope, routes.backups.create);
+    router.get ('/api/v1/backups/:backupId',   appsScope, routes.user.requireAdmin, routes.backups.download);
+
+    // disable server timeout. we use the timeout middleware to handle timeouts on a route level
+    httpServer.setTimeout(0);
 
     // upgrade handler
     httpServer.on('upgrade', function (req, socket, head) {
@@ -222,6 +236,7 @@ function initializeInternalExpressSync() {
     // internal routes
     router.post('/api/v1/backup', routes.internal.backup);
     router.post('/api/v1/update', routes.internal.update);
+    router.post('/api/v1/retire', routes.internal.retire);
 
     return httpServer;
 }

src/settings.js

@@ -29,6 +29,9 @@ exports = module.exports = {
     getTlsConfig: getTlsConfig,
     setTlsConfig: setTlsConfig,
 
+    getUpdateConfig: getUpdateConfig,
+    setUpdateConfig: setUpdateConfig,
+
     getDefaultSync: getDefaultSync,
     getAll: getAll,
 
@@ -39,6 +42,7 @@ exports = module.exports = {
     DNS_CONFIG_KEY: 'dns_config',
     BACKUP_CONFIG_KEY: 'backup_config',
     TLS_CONFIG_KEY: 'tls_config',
+    UPDATE_CONFIG_KEY: 'update_config',
 
     events: new (require('events').EventEmitter)()
 };
@@ -62,6 +66,7 @@ var gDefaults = (function () {
     result[exports.DNS_CONFIG_KEY] = { };
     result[exports.BACKUP_CONFIG_KEY] = { };
     result[exports.TLS_CONFIG_KEY] = { provider: 'caas' };
+    result[exports.UPDATE_CONFIG_KEY] = { prerelease: false };
 
     return result;
 })();
@@ -323,6 +328,30 @@ function setBackupConfig(backupConfig, callback) {
     });
 }
 
+function getUpdateConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settingsdb.get(exports.UPDATE_CONFIG_KEY, function (error, value) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.UPDATE_CONFIG_KEY]);
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        callback(null, JSON.parse(value)); // { prerelease }
+    });
+}
+
+function setUpdateConfig(updateConfig, callback) {
+    assert.strictEqual(typeof updateConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    settingsdb.set(exports.UPDATE_CONFIG_KEY, JSON.stringify(updateConfig), function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        exports.events.emit(exports.UPDATE_CONFIG_KEY, updateConfig);
+
+        callback(null);
+    });
+}
+
 function getDefaultSync(name) {
     assert.strictEqual(typeof name, 'string');
 

src/simpleauth.js

@@ -45,17 +45,20 @@ function loginLogic(clientId, username, password, callback) {
             apps.get(clientObject.appId, function (error, appObject) {
                 if (error) return callback(error);
 
-                if (!apps.hasAccessTo(appObject, userObject)) return callback(new AppsError(AppsError.ACCESS_DENIED));
-
-                var accessToken = tokendb.generateToken();
-                var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
-
-                tokendb.add(accessToken, tokendb.PREFIX_USER + userObject.id, clientId, expires, clientObject.scope, function (error) {
+                apps.hasAccessTo(appObject, userObject, function (error, access) {
                     if (error) return callback(error);
+                    if (!access) return callback(new AppsError(AppsError.ACCESS_DENIED));
 
-                    debug('login: new access token for client %s and user %s: %s', clientId, username, accessToken);
+                    var accessToken = tokendb.generateToken();
+                    var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-                    callback(null, { accessToken: accessToken, user: userObject });
+                    tokendb.add(accessToken, tokendb.PREFIX_USER + userObject.id, clientId, expires, clientObject.scope, function (error) {
+                        if (error) return callback(error);
+
+                        debug('login: new access token for client %s and user %s: %s', clientId, username, accessToken);
+
+                        callback(null, { accessToken: accessToken, user: userObject });
+                    });
                 });
             });
         });

src/storage/caas.js

@@ -77,7 +77,7 @@ function getSignedUploadUrl(backupConfig, filename, callback) {
 
         var url = s3.getSignedUrl('putObject', params);
 
-        callback(null, { url : url, sessionToken: credentials.sessionToken });
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
     });
 }
 

src/storage/s3.js

@@ -53,7 +53,8 @@ function getAllPaged(backupConfig, page, perPage, callback) {
             var results = data.Contents.map(function (backup) {
                 return {
                     creationTime: backup.LastModified,
-                    restoreKey: backup.Key.slice(backupConfig.prefix.length + 1)
+                    restoreKey: backup.Key.slice(backupConfig.prefix.length + 1),
+                    dependsOn: [] // FIXME empty dependsOn is wrong and version property is missing!!
                 };
             });
 

src/subdomains.js

@@ -44,6 +44,7 @@ SubdomainError.EXTERNAL_ERROR = 'External error';
 SubdomainError.STILL_BUSY = 'Still busy';
 SubdomainError.MISSING_CREDENTIALS = 'Missing credentials';
 SubdomainError.INTERNAL_ERROR = 'Missing credentials';
+SubdomainError.ACCESS_DENIED = 'Access denied';
 
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {

src/taskmanager.js

@@ -4,7 +4,12 @@ exports = module.exports = {
     initialize: initialize,
     uninitialize: uninitialize,
 
-    restartAppTask: restartAppTask
+    stopAppTask: stopAppTask,
+    startAppTask: startAppTask,
+    restartAppTask: restartAppTask,
+
+    stopPendingTasks: stopPendingTasks,
+    waitForPendingTasks: waitForPendingTasks
 };
 
 var appdb = require('./appdb.js'),
@@ -14,6 +19,7 @@ var appdb = require('./appdb.js'),
     cloudron = require('./cloudron.js'),
     debug = require('debug')('box:taskmanager'),
     locker = require('./locker.js'),
+    util = require('util'),
     _ = require('underscore');
 
 var gActiveTasks = { };
@@ -47,6 +53,24 @@ function uninitialize(callback) {
     async.eachSeries(Object.keys(gActiveTasks), stopAppTask, callback);
 }
 
+function stopPendingTasks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gPendingTasks = [];
+
+    async.eachSeries(Object.keys(gActiveTasks), stopAppTask, callback);
+}
+
+function waitForPendingTasks(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    function checkTasks() {
+        if (Object.keys(gActiveTasks).length === 0 && gPendingTasks.length === 0) return callback();
+        setTimeout(checkTasks, 1000);
+    }
+
+    checkTasks();
+}
 
 // resume app installs and uninstalls
 function resumeTasks(callback) {
@@ -59,7 +83,7 @@ function resumeTasks(callback) {
             if (app.installationState === appdb.ISTATE_INSTALLED && app.runState === appdb.RSTATE_RUNNING) return;
 
             debug('Creating process for %s (%s) with state %s', app.location, app.id, app.installationState);
-            startAppTask(app.id);
+            startAppTask(app.id, NOOP_CALLBACK);
         });
 
         callback(null);
@@ -71,17 +95,21 @@ function startNextTask() {
 
     assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
-    startAppTask(gPendingTasks.shift());
+    startAppTask(gPendingTasks.shift(), NOOP_CALLBACK);
 }
 
-function startAppTask(appId) {
+function startAppTask(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert(!(appId in gActiveTasks));
+    assert.strictEqual(typeof callback, 'function');
+
+    if (appId in gActiveTasks) {
+        return callback(new Error(util.format('Task for %s is already active', appId)));
+    }
 
     if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug('Reached concurrency limit, queueing task for %s', appId);
         gPendingTasks.push(appId);
-        return;
+        return callback();
     }
 
     var lockError = locker.recursiveLock(locker.OP_APPTASK);
@@ -89,9 +117,10 @@ function startAppTask(appId) {
     if (lockError) {
         debug('Locked for another operation, queueing task for %s', appId);
         gPendingTasks.push(appId);
-        return;
+        return callback();
     }
 
+    // when parent process dies, apptask processes are killed because KillMode=control-group in systemd unit file
     gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
 
     var pid = gActiveTasks[appId].pid;
@@ -106,6 +135,8 @@ function startAppTask(appId) {
         delete gActiveTasks[appId];
         locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task
     });
+
+    callback();
 }
 
 function stopAppTask(appId, callback) {
@@ -115,7 +146,7 @@ function stopAppTask(appId, callback) {
     if (gActiveTasks[appId]) {
         debug('stopAppTask : Killing existing task of %s with pid %s', appId, gActiveTasks[appId].pid);
         gActiveTasks[appId].once('exit', function () { callback(); });
-        gActiveTasks[appId].kill(); // this will end up calling the 'exit' handler
+        gActiveTasks[appId].kill('SIGKILL'); // this will end up calling the 'exit' handler
         return;
     }
 

src/test/apps-test.js

@@ -13,15 +13,54 @@ var appdb = require('../appdb.js'),
     config = require('../config.js'),
     constants = require('../constants.js'),
     database = require('../database.js'),
-    expect = require('expect.js');
+    expect = require('expect.js'),
+    groups = require('../groups.js'),
+    hat = require('hat'),
+    userdb = require('../userdb.js');
 
 describe('Apps', function () {
+    var ADMIN_0 = {
+        id: 'admin123',
+        username: 'admin123',
+        password: 'secret',
+        email: 'admin@me.com',
+        salt: 'morton',
+        createdAt: 'sometime back',
+        modifiedAt: 'now',
+        resetToken: hat(256),
+        displayName: ''
+    };
+
+    var USER_0 = {
+        id: 'uuid213',
+        username: 'uuid213',
+        password: 'secret',
+        email: 'safe@me.com',
+        salt: 'morton',
+        createdAt: 'sometime back',
+        modifiedAt: 'now',
+        resetToken: hat(256),
+        displayName: ''
+    };
+
+    var USER_1 = {
+        id: 'uuid2134',
+        username: 'uuid2134',
+        password: 'secret',
+        email: 'safe1@me.com',
+        salt: 'morton',
+        createdAt: 'sometime back',
+        modifiedAt: 'now',
+        resetToken: hat(256),
+        displayName: ''
+    };
+
+    var GROUP_0 = 'somegroup';
+    var GROUP_1 = 'someothergroup';
+
     var APP_0 = {
         id: 'appid-0',
         appStoreId: 'appStoreId-0',
-        installationState: appdb.ISTATE_PENDING_INSTALL,
-        installationProgress: null,
-        runState: null,
         location: 'some-location-0',
         manifest: {
             version: '0.1', dockerImage: 'docker/app0', healthCheckPath: '/', httpPort: 80, title: 'app0',
@@ -32,19 +71,51 @@ describe('Apps', function () {
                 }
             }
         },
-        httpPort: null,
-        containerId: null,
         portBindings: { PORT: 5678 },
-        healthy: null,
         accessRestriction: null,
-        oauthProxy: false
+        memoryLimit: 0
+    };
+
+    var APP_1 = {
+        id: 'appid-1',
+        appStoreId: 'appStoreId-1',
+        location: 'some-location-1',
+        manifest: {
+            version: '0.1', dockerImage: 'docker/app1', healthCheckPath: '/', httpPort: 80, title: 'app1',
+            tcpPorts: {}
+        },
+        portBindings: null,
+        accessRestriction: { users: [ 'someuser' ], groups: [ GROUP_0 ] },
+        memoryLimit: 0
+    };
+
+    var APP_2 = {
+        id: 'appid-2',
+        appStoreId: 'appStoreId-2',
+        location: 'some-location-2',
+        manifest: {
+            version: '0.1', dockerImage: 'docker/app2', healthCheckPath: '/', httpPort: 80, title: 'app2',
+            tcpPorts: {}
+        },
+        portBindings: null,
+        accessRestriction: { users: [ 'someuser', USER_0.id ], groups: [ GROUP_1 ] },
+        memoryLimit: 0
     };
 
     before(function (done) {
         async.series([
             database.initialize,
             database._clear,
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.oauthProxy)
+            userdb.add.bind(null, ADMIN_0.id, ADMIN_0),
+            userdb.add.bind(null, USER_0.id, USER_0),
+            userdb.add.bind(null, USER_1.id, USER_1),
+            groups.create.bind(null, GROUP_0),
+            groups.create.bind(null, GROUP_1),
+            groups.addMember.bind(null, groups.ADMIN_GROUP_ID, ADMIN_0.id),
+            groups.addMember.bind(null, GROUP_0, USER_1.id),
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit)
         ], done);
     });
 
@@ -125,6 +196,7 @@ describe('Apps', function () {
                 expect(app).to.be.ok();
                 expect(app.iconUrl).to.be(null);
                 expect(app.fqdn).to.eql(APP_0.location + '-' + config.fqdn());
+                expect(app.memoryLimit).to.eql(0);
                 done();
             });
         });
@@ -182,24 +254,91 @@ describe('Apps', function () {
     });
 
     describe('hasAccessTo', function () {
-        it('returns true for unrestricted access', function () {
-            expect(apps.hasAccessTo({ accessRestriction: null }, { id: 'someuser' })).to.equal(true);
+        it('returns true for unrestricted access', function (done) {
+            apps.hasAccessTo({ accessRestriction: null }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(true);
+                done();
+            });
         });
 
-        it('returns true for allowed user', function () {
-            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'someuser' ] } }, { id: 'someuser' })).to.equal(true);
+        it('returns true for allowed user', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'someuser' ] } }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(true);
+                done();
+            });
         });
 
-        it('returns true for allowed user with multiple allowed', function () {
-            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'someuser', 'anotheruser' ] } }, { id: 'someuser' })).to.equal(true);
+        it('returns true for allowed user with multiple allowed', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'someuser', 'anotheruser' ] } }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(true);
+                done();
+            });
         });
 
-        it('returns false for not allowed user', function () {
-            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'foo' ] } }, { id: 'someuser' })).to.equal(false);
+        it('returns false for not allowed user', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'foo' ] } }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(false);
+                done();
+            });
         });
 
-        it('returns false for not allowed user with multiple allowed', function () {
-            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'anotheruser' ] } }, { id: 'someuser' })).to.equal(false);
+        it('returns false for not allowed user with multiple allowed', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'anotheruser' ] } }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(false);
+                done();
+            });
+        });
+
+        it('returns false for no group or user', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ ] } }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(false);
+                done();
+            });
+        });
+
+        it('returns false for invalid group or user', function (done) {
+            apps.hasAccessTo({ accessRestriction: { users: [ ], groups: [ 'nop' ] } }, { id: 'someuser' }, function (error, access) {
+                expect(error).to.be(null);
+                expect(access).to.be(false);
+                done();
+            });
+        });
+    });
+
+    describe('getAllByUser', function () {
+        it('succeeds for USER_0', function (done) {
+            apps.getAllByUser(USER_0, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result.length).to.equal(2);
+                expect(result[0].id).to.equal(APP_0.id);
+                expect(result[1].id).to.equal(APP_2.id);
+                done();
+            });
+        });
+
+        it('succeeds for USER_1', function (done) {
+            apps.getAllByUser(USER_1, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result.length).to.equal(2);
+                expect(result[0].id).to.equal(APP_0.id);
+                expect(result[1].id).to.equal(APP_1.id);
+                done();
+            });
+        });
+
+        it('succeeds with admin not being special', function (done) {
+            apps.getAllByUser(ADMIN_0, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result.length).to.equal(1);
+                expect(result[0].id).to.equal(APP_0.id);
+                done();
+            });
         });
     });
 });

src/test/apptask-test.js

@@ -60,8 +60,8 @@ var APP = {
     httpPort: 4567,
     portBindings: null,
     accessRestriction: null,
-    oauthProxy: false,
-    dnsRecordId: 'someDnsRecordId'
+    dnsRecordId: 'someDnsRecordId',
+    memoryLimit: 0
 };
 
  var awsHostedZones = {
@@ -84,7 +84,7 @@ describe('apptask', function () {
         config.set('version', '0.5.0');
         async.series([
             database.initialize,
-            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, APP.oauthProxy),
+            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, APP.memoryLimit),
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
             settings.setTlsConfig.bind(null, { provider: 'caas' })
         ], done);