Make all json blobs in db TEXT fields

This reverts commit 3b5e30f922.

.gitignore

@@ -4,10 +4,6 @@ docs/
 webadmin/dist/
 setup/splash/website/
 
-# vim swam files
+# vim swap files
 *.swp
 
-# supervisor
-supervisord.pid
-supervisord.log
-

README.md

@@ -4,7 +4,6 @@ The Box
 Development setup
 -----------------
 * sudo useradd -m yellowtent
-** This dummy user is required for supervisor 'box' configs
 ** Add admin-localhost as 127.0.0.1 in /etc/hosts
 ** All apps will be installed as hypened-subdomains of localhost. You should add
    hyphened-subdomains of your apps into /etc/hosts

app.js

@@ -1,47 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var server = require('./src/server.js'),
-    ldap = require('./src/ldap.js'),
-    config = require('./src/config.js');
-
-console.log();
-console.log('==========================================');
-console.log(' Cloudron will use the following settings ');
-console.log('==========================================');
-console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
-console.log(' Version:                        ', config.version());
-console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore token:                 ', config.token());
-console.log(' Appstore API server origin:     ', config.apiServerOrigin());
-console.log(' Appstore Web server origin:     ', config.webServerOrigin());
-console.log();
-console.log('==========================================');
-console.log();
-
-server.start(function (err) {
-    if (err) {
-        console.error('Error starting server', err);
-        process.exit(1);
-    }
-
-    console.log('Server listening on port ' + config.get('port'));
-
-    ldap.start(function (error) {
-        if (error) {
-            console.error('Error LDAP starting server', err);
-            process.exit(1);
-        }
-
-        console.log('LDAP server listen on port ' + config.get('ldapPort'));
-    });
-});
-
-var NOOP_CALLBACK = function () { };
-
-process.on('SIGINT', function () { server.stop(NOOP_CALLBACK); });
-process.on('SIGTERM', function () { server.stop(NOOP_CALLBACK); });

box.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
+var appHealthMonitor = require('./src/apphealthmonitor.js'),
+    async = require('async'),
+    config = require('./src/config.js'),
+    ldap = require('./src/ldap.js'),
+    simpleauth = require('./src/simpleauth.js'),
+    oauthproxy = require('./src/oauthproxy.js'),
+    server = require('./src/server.js');
+
+console.log();
+console.log('==========================================');
+console.log(' Cloudron will use the following settings ');
+console.log('==========================================');
+console.log();
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
+console.log(' Version:                        ', config.version());
+console.log(' Admin Origin:                   ', config.adminOrigin());
+console.log(' Appstore token:                 ', config.token());
+console.log(' Appstore API server origin:     ', config.apiServerOrigin());
+console.log(' Appstore Web server origin:     ', config.webServerOrigin());
+console.log();
+console.log('==========================================');
+console.log();
+
+async.series([
+    server.start,
+    ldap.start,
+    simpleauth.start,
+    appHealthMonitor.start,
+    oauthproxy.start
+], function (error) {
+    if (error) {
+        console.error('Error starting server', error);
+        process.exit(1);
+    }
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});

crashnotifier.js

@@ -2,20 +2,12 @@
 
 'use strict';
 
-// WARNING This is a supervisor eventlistener!
-//         The communication happens via stdin/stdout
-//         !! No console.log() allowed
-//         !! Do not set DEBUG
-
 var assert = require('assert'),
     mailer = require('./src/mailer.js'),
     safe = require('safetydance'),
-    supervisor = require('supervisord-eventlistener'),
     path = require('path'),
     util = require('util');
 
-var gLastNotifyTime = {};
-var gCooldownTime = 1000 * 60  * 5; // 5 min
 var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
 
 function collectLogs(program, callback) {
@@ -26,28 +18,25 @@ function collectLogs(program, callback) {
     callback(null, logs);
 }
 
-supervisor.on('PROCESS_STATE_EXITED', function (headers, data) {
-    if (data.expected === '1') return console.error('Normal app %s exit', data.processname);
-
-    console.error('%s exited unexpectedly', data.processname);
-
-    collectLogs(data.processname, function (error, result) {
+function sendCrashNotification(processName) {
+    collectLogs(processName, function (error, result) {
         if (error) {
             console.error('Failed to collect logs.', error);
             result = util.format('Failed to collect logs.', error);
         }
 
-        if (!gLastNotifyTime[data.processname] || gLastNotifyTime[data.processname] < Date.now() - gCooldownTime) {
-            console.error('Send mail.');
-            mailer.sendCrashNotification(data.processname, result);
-            gLastNotifyTime[data.processname] = Date.now();
-        } else {
-            console.error('Do not send mail, already sent one recently.');
-        }
+        console.log('Sending crash notification email for', processName);
+        mailer.sendCrashNotification(processName, result);
     });
-});
+}
 
-mailer.initialize(function () {
-    supervisor.listen(process.stdin, process.stdout);
-    console.error('Crashnotifier listening...');
-});
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendCrashNotification(processName);
+}
+
+main();

janitor.js

@@ -1,70 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var assert = require('assert'),
-    debug = require('debug')('box:janitor'),
-    async = require('async'),
-    tokendb = require('./src/tokendb.js'),
-    authcodedb = require('./src/authcodedb.js'),
-    database = require('./src/database.js');
-
-var TOKEN_CLEANUP_INTERVAL = 30000;
-
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.initialize
-    ], callback);
-}
-
-function cleanupExpiredTokens(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired tokens.', result);
-
-        callback(null);
-    });
-}
-
-function cleanupExpiredAuthCodes(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    authcodedb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired authcodes.', result);
-
-        callback(null);
-    });
-}
-
-function run() {
-    cleanupExpiredTokens(function (error) {
-        if (error) console.error(error);
-
-        cleanupExpiredAuthCodes(function (error) {
-            if (error) console.error(error);
-
-            setTimeout(run, TOKEN_CLEANUP_INTERVAL);
-        });
-    });
-}
-
-if (require.main === module) {
-    initialize(function (error) {
-        if (error) {
-            console.error('janitor task exiting with error', error);
-            process.exit(1);
-        }
-
-        run();
-    });
-}
-

migrations/20151013073519-apps-add-oauthProxy.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151015232915-clients-add-type.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'DELETE FROM clients'),
+        db.runSql.bind(db, 'ALTER TABLE clients ADD COLUMN type VARCHAR(16) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE clients DROP COLUMN type', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151016131005-apps-rename-accessRestriction.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestriction accessRestrictionJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestrictionJson accessRestriction VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151113091257-apps-alter-manifestJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151126111337-apps-alter-jsons.js

@@ -0,0 +1,19 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson TEXT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson VARCHAR(2048)')
+    ], callback);
+};

migrations/schema.sql

@@ -29,8 +29,9 @@ CREATE TABLE IF NOT EXISTS tokens(
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE,
+    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
     appId VARCHAR(128) NOT NULL,
+    type VARCHAR(16) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
@@ -44,15 +45,19 @@ CREATE TABLE IF NOT EXISTS apps(
     runState VARCHAR(512),
     health VARCHAR(128),
     containerId VARCHAR(128),
-    manifestJson VARCHAR(2048),
+    manifestJson TEXT,
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestriction VARCHAR(512),
+    accessRestrictionJson TEXT,
+    oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
     lastBackupId VARCHAR(128),
-    lastBackupConfigJson VARCHAR(2048), // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+
+    oldConfigJson TEXT, // used to pass old config for apptask
+
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(

oauthproxy.js

@@ -1,185 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var express = require('express'),
-    url = require('url'),
-    uuid = require('node-uuid'),
-    async = require('async'),
-    superagent = require('superagent'),
-    assert = require('assert'),
-    debug = require('debug')('box:proxy'),
-    proxy = require('proxy-middleware'),
-    session = require('cookie-session'),
-    database = require('./src/database.js'),
-    appdb = require('./src/appdb.js'),
-    clientdb = require('./src/clientdb.js'),
-    config = require('./src/config.js'),
-    http = require('http');
-
-// Allow self signed certs!
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
-
-var gSessions = {};
-var gProxyMiddlewareCache = {};
-var gApp = express();
-var gHttpServer = http.createServer(gApp);
-
-var CALLBACK_URI = '/callback';
-var PORT = 4000;
-
-function startServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    gHttpServer.on('error', console.error);
-
-    gApp.use(session({
-        keys: ['blue', 'cheese', 'is', 'something']
-    }));
-
-    // ensure we have a in memory store for the session to cache client information
-    gApp.use(function (req, res, next) {
-        assert.strictEqual(typeof req.session, 'object');
-
-        if (!req.session.id || !gSessions[req.session.id]) {
-            req.session.id = uuid.v4();
-            gSessions[req.session.id] = {};
-        }
-
-        // attach the session data to the requeset
-        req.sessionData = gSessions[req.session.id];
-
-        next();
-    });
-
-    gApp.use(function verifySession(req, res, next) {
-        assert.strictEqual(typeof req.sessionData, 'object');
-
-        if (!req.sessionData.accessToken) {
-            req.authenticated = false;
-            return next();
-        }
-
-        superagent.get(config.adminOrigin() + '/api/v1/profile').query({ access_token: req.sessionData.accessToken}).end(function (error, result) {
-            if (error) {
-                console.error(error);
-                req.authenticated = false;
-            } else if (result.statusCode !== 200) {
-                req.sessionData.accessToken = null;
-                req.authenticated = false;
-            } else {
-                req.authenticated = true;
-            }
-
-            next();
-        });
-    });
-
-    gApp.use(function (req, res, next) {
-        // proceed if we are authenticated
-        if (req.authenticated) return next();
-
-        if (req.path === CALLBACK_URI && req.sessionData.returnTo) {
-            // exchange auth code for an access token
-            var query = {
-                response_type: 'token',
-                client_id: req.sessionData.clientId
-            };
-
-            var data = {
-                grant_type: 'authorization_code',
-                code: req.query.code,
-                redirect_uri: req.sessionData.returnTo,
-                client_id: req.sessionData.clientId,
-                client_secret: req.sessionData.clientSecret
-            };
-
-            superagent.post(config.adminOrigin() + '/api/v1/oauth/token').query(query).send(data).end(function (error, result) {
-                if (error) {
-                    console.error(error);
-                    return res.send(500, 'Unable to contact the oauth server.');
-                }
-                if (result.statusCode !== 200) {
-                    console.error('Failed to exchange auth code for a token.', result.statusCode, result.body);
-                    return res.send(500, 'Failed to exchange auth code for a token.');
-                }
-
-                req.sessionData.accessToken = result.body.access_token;
-
-                debug('user verified.');
-
-                // now redirect to the actual initially requested URL
-                res.redirect(req.sessionData.returnTo);
-            });
-        } else {
-            var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
-
-            if (!Number.isFinite(port)) {
-                console.error('Failed to parse nginx proxy header to get app port.');
-                return res.send(500, 'Routing error. No forwarded port.');
-            }
-
-            debug('begin verifying user for app on port %s.', port);
-
-            appdb.getByHttpPort(port, function (error, result) {
-                if (error) {
-                    console.error('Unknown app.', error);
-                    return res.send(500, 'Unknown app.');
-                }
-
-                clientdb.getByAppId('proxy-' + result.id, function (error, result) {
-                    if (error) {
-                        console.error('Unkonwn OAuth client.', error);
-                        return res.send(500, 'Unknown OAuth client.');
-                    }
-
-                    req.sessionData.port = port;
-                    req.sessionData.returnTo = result.redirectURI + req.path;
-                    req.sessionData.clientId = result.id;
-                    req.sessionData.clientSecret = result.clientSecret;
-
-                    var callbackUrl = result.redirectURI + CALLBACK_URI;
-                    var scope = 'profile,roleUser';
-                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + result.id + '&redirect_uri=' + callbackUrl + '&scope=' + scope;
-
-                    debug('begin OAuth flow for client %s.', result.name);
-
-                    // begin the OAuth flow
-                    res.redirect(oauthLogin);
-                });
-            });
-        }
-    });
-
-    gApp.use(function (req, res, next) {
-        var port = req.sessionData.port;
-
-        debug('proxy request for port %s with path %s.', port, req.path);
-
-        var proxyMiddleware = gProxyMiddlewareCache[port];
-        if (!proxyMiddleware) {
-            console.log('Adding proxy middleware for port %d', port);
-
-            proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
-            gProxyMiddlewareCache[port] = proxyMiddleware;
-        }
-
-        proxyMiddleware(req, res, next);
-    });
-
-    gHttpServer.listen(PORT, callback);
-}
-
-async.series([
-    database.initialize,
-    startServer
-], function (error) {
-    if (error) {
-        console.error('Failed to start proxy server.', error);
-        process.exit(1);
-    }
-
-    console.log('Proxy server listening...');
-});

package.json

@@ -10,15 +10,14 @@
     "type": "git"
   },
   "engines": [
-    "node >= 0.12.0"
+    "node >=4.0.0 <=4.1.1"
   ],
-  "bin": {
-    "cloudron": "./app.js"
-  },
   "dependencies": {
     "async": "^1.2.1",
+    "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^1.6.0",
+    "bytes": "^2.1.0",
+    "cloudron-manifestformat": "^2.0.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
@@ -59,16 +58,15 @@
     "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
     "superagent": "~0.21.0",
-    "supererror": "^0.7.0",
-    "supervisord-eventlistener": "^0.1.0",
+    "supererror": "^0.7.1",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "underscore": "^1.7.0",
     "valid-url": "^1.0.9",
-    "validator": "^3.30.0"
+    "validator": "^3.30.0",
+    "x509": "^0.2.2"
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "^2.1.10",
     "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",
@@ -83,10 +81,12 @@
     "gulp-uglify": "^1.1.0",
     "hock": "~1.2.0",
     "istanbul": "*",
+    "js2xmlparser": "^1.0.0",
     "mocha": "*",
     "nock": "^2.6.0",
     "node-sass": "^3.0.0-alpha.0",
     "redis": "^0.12.1",
+    "request": "^2.65.0",
     "sinon": "^1.12.2",
     "yargs": "^3.15.0"
   },

setup/DESIGN.md

@@ -16,7 +16,7 @@ and replace it with a new one for an update.
 
 Because we do not package things as Docker yet, we should be careful
 about the code here. We have to expect remains of an older setup code.
-For example, older supervisor or nginx configs might be around.
+For example, older systemd or nginx configs might be around.
 
 The config directory is _part_ of the container and is not a VOLUME.
 Which is to say that the files will be nuked from one update to the next.
@@ -40,7 +40,7 @@ version (see below) or the mysql/postgresql data etc.
 
   * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
 
-  * supervisor is then started
+  * box services are then started
 
 setup_infra.sh
 This setups containers like graphite, mail and the addons containers.

setup/INFRA_VERSION

@@ -3,15 +3,21 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=8
+INFRA_VERSION=21
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.3.1
-MYSQL_IMAGE=cloudron/mysql:0.3.2
-POSTGRESQL_IMAGE=cloudron/postgresql:0.3.1
-MONGODB_IMAGE=cloudron/mongodb:0.3.1
-REDIS_IMAGE=cloudron/redis:0.3.1 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.3.1
-GRAPHITE_IMAGE=cloudron/graphite:0.3.3
+BASE_IMAGE=cloudron/base:0.8.0
+MYSQL_IMAGE=cloudron/mysql:0.8.0
+POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
+MONGODB_IMAGE=cloudron/mongodb:0.8.0
+REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
+MAIL_IMAGE=cloudron/mail:0.9.0
+GRAPHITE_IMAGE=cloudron/graphite:0.8.0
 
+MYSQL_REPO=cloudron/mysql
+POSTGRESQL_REPO=cloudron/postgresql
+MONGODB_REPO=cloudron/mongodb
+REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
+MAIL_REPO=cloudron/mail
+GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -16,6 +16,8 @@ arg_tls_key=""
 arg_token=""
 arg_version=""
 arg_web_server_origin=""
+arg_backup_config=""
+arg_dns_config=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -35,12 +37,18 @@ EOF
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
 
-        arg_restore_url=$(echo "$2" | $json restoreUrl)
+        arg_restore_url=$(echo "$2" | $json restore.url)
         [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
 
-        arg_restore_key=$(echo "$2" | $json restoreKey)
+        arg_restore_key=$(echo "$2" | $json restore.key)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
+        arg_backup_config=$(echo "$2" | $json backupConfig)
+        [[ "${arg_backup_config}" == "null" ]] && arg_backup_config=""
+
+        arg_dns_config=$(echo "$2" | $json dnsConfig)
+        [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
+
         shift 2
         ;;
     --) break;;

setup/container.sh

@@ -13,22 +13,24 @@ readonly DATA_DIR="/home/yellowtent/data"
 rm -rf "${CONFIG_DIR}"
 sudo -u yellowtent mkdir "${CONFIG_DIR}"
 
-########## logrotate (default ubuntu runs this daily)
-rm -rf /etc/logrotate.d/*
-cp -r "${container_files}/logrotate/." /etc/logrotate.d/
-
-########## supervisor
-rm -rf /etc/supervisor/*
-cp -r "${container_files}/supervisor/." /etc/supervisor/
+########## systemd
+rm -f /etc/systemd/system/janitor.*
+cp -r "${container_files}/systemd/." /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable cloudron.target
 
 ########## sudoers
-rm /etc/sudoers.d/*
+rm -f /etc/sudoers.d/*
 cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
 
 ########## collectd
 rm -rf /etc/collectd
 ln -sfF "${DATA_DIR}/collectd" /etc/collectd
 
+########## apparmor docker profile
+cp "${container_files}/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
+systemctl restart apparmor
+
 ########## nginx
 # link nginx config to system config
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx

setup/container/docker-cloudron-app.apparmor

@@ -0,0 +1,32 @@
+#include <tunables/global>
+
+
+profile docker-cloudron-app flags=(attach_disconnected,mediate_deleted) {
+
+  #include <abstractions/base>
+
+  ptrace peer=@{profile_name},
+
+  network,
+  capability,
+  file,
+  umount,
+
+  deny @{PROC}/sys/fs/** wklx,
+  deny @{PROC}/sysrq-trigger rwklx,
+  deny @{PROC}/mem rwklx,
+  deny @{PROC}/kmem rwklx,
+  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
+  deny @{PROC}/sys/kernel/*/** wklx,
+
+  deny mount,
+
+  deny /sys/[^f]*/** wklx,
+  deny /sys/f[^s]*/** wklx,
+  deny /sys/fs/[^c]*/** wklx,
+  deny /sys/fs/c[^g]*/** wklx,
+  deny /sys/fs/cg[^r]*/** wklx,
+  deny /sys/firmware/efi/efivars/** rwklx,
+  deny /sys/kernel/security/** rwklx,
+}
+

setup/container/logrotate/cloudron

@@ -1,6 +0,0 @@
-/var/log/cloudron/*log {
-    missingok
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/logrotate/supervisor

@@ -1,7 +0,0 @@
-/var/log/supervisor/*log {
-    missingok
-    copytruncate
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/supervisor/conf.d/apphealthtask.conf

@@ -1,10 +0,0 @@
-[program:apphealthtask]
-command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/box.conf

@@ -1,10 +0,0 @@
-[program:box]
-command=/usr/bin/node "/home/yellowtent/box/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/crashnotifier.conf

@@ -1,11 +0,0 @@
-[eventlistener:crashnotifier]
-command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
-events=PROCESS_STATE
-autostart=true
-autorestart=true
-redirect_stderr=false
-stderr_logfile=/var/log/supervisor/crashnotifier.log
-stderr_logfile_maxbytes=50MB
-stderr_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/janitor.conf

@@ -1,10 +0,0 @@
-[program:janitor]
-command=/usr/bin/node "/home/yellowtent/box/janitor.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/janitor.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/oauthproxy.conf

@@ -1,10 +0,0 @@
-[program:oauthproxy]
-command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/oauthproxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/supervisord.conf

@@ -1,33 +0,0 @@
-; supervisor config file
-
-; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[supervisord]
-logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
-pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-nodaemon = false
-childlogdir = /var/log/supervisor/
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=http://127.0.0.1:9001
-
-; The [include] section can just contain the "files" setting.  This
-; setting can list multiple files (separated by whitespace or
-; newlines).  It can also contain wildcards.  The filenames are
-; interpreted as relative to this file.  Included files *cannot*
-; include files themselves.
-
-[include]
-files = conf.d/*.conf
-

setup/container/systemd/box.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smart Cloud
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service
+After=box.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/splashpage.sh

@@ -29,10 +29,10 @@ infra_version="none"
 if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi
 
 echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"

setup/start.sh

@@ -38,6 +38,7 @@ set_progress "10" "Ensuring directories"
 # keep these in sync with paths.js
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
 mkdir -p "${DATA_DIR}/box/appicons"
+mkdir -p "${DATA_DIR}/box/certs"
 mkdir -p "${DATA_DIR}/box/mail"
 mkdir -p "${DATA_DIR}/graphite"
 
@@ -105,15 +106,27 @@ ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs
     -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
 
 # generate these for update code paths as well to overwrite splash
+admin_cert_file="cert/host.cert"
+admin_key_file="cert/host.key"
+if [[ -f "${DATA_DIR}/box/certs/admin.cert" && -f "${DATA_DIR}/box/certs/admin.key" ]]; then
+    admin_cert_file="${DATA_DIR}/box/certs/admin.cert"
+    admin_key_file="${DATA_DIR}/box/certs/admin.key"
+fi
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
 mkdir -p "${DATA_DIR}/nginx/cert"
-echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
-echo "${arg_tls_key}" > ${DATA_DIR}/nginx/cert/host.key
+if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
+    cp "${DATA_DIR}/box/certs/host.cert" "${DATA_DIR}/nginx/cert/host.cert"
+    cp "${DATA_DIR}/box/certs/host.key" "${DATA_DIR}/nginx/cert/host.key"
+else
+    echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
+    echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
+fi
 
 set_progress "33" "Changing ownership"
 chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+chown "${USER}:${USER}" "${DATA_DIR}"
 
 set_progress "40" "Setting up infra"
 ${script_dir}/start/setup_infra.sh "${arg_fqdn}"
@@ -150,35 +163,39 @@ cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
 CONF_END
 EOF
 
+# Add Backup Configuration
+if [[ ! -z "${arg_backup_config}" ]]; then
+    echo "Add Backup Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
+fi
+
+# Add DNS Configuration
+if [[ ! -z "${arg_dns_config}" ]]; then
+    echo "Add DNS Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
+fi
+
 # Add webadmin oauth client
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
 echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-echo "Add localhost test oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+echo "Add localhost test oauth client"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "80" "Reloading supervisor"
-# looks like restarting supervisor completely is the only way to reload it
-service supervisor stop || true
+set_progress "80" "Starting Cloudron"
+systemctl start cloudron.target
 
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
+sleep 2 # give systemd sometime to start the processes
 
 set_progress "85" "Reloading nginx"
 nginx -s reload

setup/start/collectd.conf

@@ -220,7 +220,7 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin processes>
-   ProcessMatch "app" "node app.js"
+   ProcessMatch "app" "node box.js"
 </Plugin>
 
 <Plugin swap>

setup/start/nginx/appconfig.ejs

@@ -10,8 +10,8 @@ server {
 
     ssl                  on;
     # paths are relative to prefix and not to this file
-    ssl_certificate      cert/host.cert;
-    ssl_certificate_key  cert/host.key;
+    ssl_certificate      <%= certFilePath %>;
+    ssl_certificate_key  <%= keyFilePath %>;
     ssl_session_timeout  5m;
     ssl_session_cache shared:SSL:50m;
 
@@ -37,7 +37,8 @@ server {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
 
-    error_page 500 502 503 504 @appstatus;
+    # only serve up the status page if we get proxy gateway errors
+    error_page 502 503 504 @appstatus;
     location @appstatus {
         return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
     }
@@ -69,7 +70,7 @@ server {
         }
 
 <% } else if ( endpoint === 'oauthproxy' ) { %>
-        proxy_pass http://127.0.0.1:4000;
+        proxy_pass http://127.0.0.1:3003;
         proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
 <% } else if ( endpoint === 'app' ) { %>
         proxy_pass http://127.0.0.1:<%= port %>;

setup/start/setup_infra.sh

@@ -28,21 +28,32 @@ fi
 
 # graphite
 graphite_container_id=$(docker run --restart=always -d --name="graphite" \
+    -m 75m \
+    --memory-swap 150m \
     -p 127.0.0.1:2003:2003 \
     -p 127.0.0.1:2004:2004 \
     -p 127.0.0.1:8000:8000 \
     -v "${DATA_DIR}/graphite:/app/data" \
+    --read-only -v /tmp -v /run \
     "${GRAPHITE_IMAGE}")
 echo "Graphite container id: ${graphite_container_id}"
+if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old graphite images"
+fi
 
-# mail
+# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
 mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -p 127.0.0.1:25:25 \
+    -m 75m \
+    --memory-swap 150m \
     -h "${arg_fqdn}" \
     -e "DOMAIN_NAME=${arg_fqdn}" \
     -v "${DATA_DIR}/box/mail:/app/data" \
+    --read-only -v /tmp -v /run \
     "${MAIL_IMAGE}")
 echo "Mail container id: ${mail_container_id}"
+if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mail images"
+fi
 
 # mysql
 mysql_addon_root_password=$(pwgen -1 -s)
@@ -52,11 +63,17 @@ readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
 readonly MYSQL_ROOT_HOST='${docker0_ip}'
 EOF
 mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mysql:/var/lib/mysql" \
     -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
     "${MYSQL_IMAGE}")
 echo "MySQL container id: ${mysql_container_id}"
+if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mysql images"
+fi
 
 # postgresql
 postgresql_addon_root_password=$(pwgen -1 -s)
@@ -64,11 +81,17 @@ cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
 readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
 EOF
 postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
     -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
     "${POSTGRESQL_IMAGE}")
 echo "PostgreSQL container id: ${postgresql_container_id}"
+if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old postgresql images"
+fi
 
 # mongodb
 mongodb_addon_root_password=$(pwgen -1 -s)
@@ -76,20 +99,31 @@ cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
 readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
 EOF
 mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
     -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
     "${MONGODB_IMAGE}")
 echo "Mongodb container id: ${mongodb_container_id}"
+if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mongodb images"
+fi
 
+# redis
+if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old redis images"
+fi
+
+# only touch apps in installed state. any other state is just resumed by the taskmanager
 if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new and restoring cloudons), download app backups
+    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
     echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore" WHERE installationState = "installed"' box
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
 else
     # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure" WHERE installationState = "installed"' box
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
 fi
 
 echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"
-

setup/stop.sh

@@ -2,14 +2,6 @@
 
 set -eu -o pipefail
 
-echo "Stopping box code"
-
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
+echo "Stopping cloudron"
 
+systemctl stop cloudron.target

src/addons.js

@@ -9,10 +9,11 @@ exports = module.exports = {
     getEnvironment: getEnvironment,
     getLinksSync: getLinksSync,
     getBindsSync: getBindsSync,
+    getContainerNamesSync: getContainerNamesSync,
 
     // exported for testing
-    _allocateOAuthCredentials: allocateOAuthCredentials,
-    _removeOAuthCredentials: removeOAuthCredentials
+    _setupOauth: setupOauth,
+    _teardownOauth: teardownOauth
 };
 
 var appdb = require('./appdb.js'),
@@ -23,64 +24,36 @@ var appdb = require('./appdb.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:addons'),
-    docker = require('./docker.js'),
+    docker = require('./docker.js').connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
     MemoryStream = require('memorystream'),
     once = require('once'),
-    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     spawn = child_process.spawn,
-    tokendb = require('./tokendb.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
-    _ = require('underscore');
+    uuid = require('node-uuid');
 
-var NOOP = function (app, callback) { return callback(); };
+var NOOP = function (app, options, callback) { return callback(); };
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
-    oauth: {
-        setup: allocateOAuthCredentials,
-        teardown: removeOAuthCredentials,
-        backup: NOOP,
-        restore: allocateOAuthCredentials
-    },
-    token: {
-        setup: allocateAccessToken,
-        teardown: removeAccessToken,
-        backup: NOOP,
-        restore: allocateAccessToken
-    },
     ldap: {
         setup: setupLdap,
         teardown: teardownLdap,
         backup: NOOP,
         restore: setupLdap
     },
-    sendmail: {
-        setup: setupSendMail,
-        teardown: teardownSendMail,
-        backup: NOOP,
-        restore: setupSendMail
-    },
-    mysql: {
-        setup: setupMySql,
-        teardown: teardownMySql,
-        backup: backupMySql,
-        restore: restoreMySql,
-    },
-    postgresql: {
-        setup: setupPostgreSql,
-        teardown: teardownPostgreSql,
-        backup: backupPostgreSql,
-        restore: restorePostgreSql
+    localstorage: {
+        setup: NOOP, // docker creates the directory for us
+        teardown: NOOP,
+        backup: NOOP, // no backup because it's already inside app data
+        restore: NOOP
     },
     mongodb: {
         setup: setupMongoDb,
@@ -88,18 +61,48 @@ var KNOWN_ADDONS = {
         backup: backupMongoDb,
         restore: restoreMongoDb
     },
+    mysql: {
+        setup: setupMySql,
+        teardown: teardownMySql,
+        backup: backupMySql,
+        restore: restoreMySql,
+    },
+    oauth: {
+        setup: setupOauth,
+        teardown: teardownOauth,
+        backup: NOOP,
+        restore: setupOauth
+    },
+    postgresql: {
+        setup: setupPostgreSql,
+        teardown: teardownPostgreSql,
+        backup: backupPostgreSql,
+        restore: restorePostgreSql
+    },
     redis: {
         setup: setupRedis,
         teardown: teardownRedis,
-        backup: NOOP, // no backup because we store redis as part of app's volume
+        backup: backupRedis,
         restore: setupRedis // same thing
     },
-    localstorage: {
-        setup: NOOP, // docker creates the directory for us
+    sendmail: {
+        setup: setupSendMail,
+        teardown: teardownSendMail,
+        backup: NOOP,
+        restore: setupSendMail
+    },
+    scheduler: {
+        setup: NOOP,
         teardown: NOOP,
-        backup: NOOP, // no backup because it's already inside app data
+        backup: NOOP,
         restore: NOOP
     },
+    simpleauth: {
+        setup: setupSimpleAuth,
+        teardown: teardownSimpleAuth,
+        backup: NOOP,
+        restore: setupSimpleAuth
+    },
     _docker: {
         setup: NOOP,
         teardown: NOOP,
@@ -129,9 +132,9 @@ function setupAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        debugApp(app, 'Setting up addon %s', addon);
+        debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);
 
-        KNOWN_ADDONS[addon].setup(app, iteratorCallback);
+        KNOWN_ADDONS[addon].setup(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -147,9 +150,9 @@ function teardownAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        debugApp(app, 'Tearing down addon %s', addon);
+        debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);
 
-        KNOWN_ADDONS[addon].teardown(app, iteratorCallback);
+        KNOWN_ADDONS[addon].teardown(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -167,7 +170,7 @@ function backupAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        KNOWN_ADDONS[addon].backup(app, iteratorCallback);
+        KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -185,7 +188,7 @@ function restoreAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        KNOWN_ADDONS[addon].restore(app, iteratorCallback);
+        KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -237,22 +240,44 @@ function getBindsSync(app, addons) {
     return binds;
 }
 
-function allocateOAuthCredentials(app, callback) {
+function getContainerNamesSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+
+    var names = [ ];
+
+    if (!addons) return names;
+
+    for (var addon in addons) {
+        switch (addon) {
+        case 'scheduler':
+            // names here depend on how scheduler.js creates containers
+            names = names.concat(Object.keys(addons.scheduler).map(function (taskName) { return app.id + '-' + taskName; }));
+            break;
+        default: break;
+        }
+    }
+
+    return names;
+}
+
+function setupOauth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-addon-' + uuid.v4();
+    var id = 'cid-' + uuid.v4();
     var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
-    var scope = 'profile,roleUser';
+    var scope = 'profile';
 
-    debugApp(app, 'allocateOAuthCredentials: id:%s clientSecret:%s', id, clientSecret);
+    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
 
-    clientdb.delByAppId('addon-' + appId, function (error) { // remove existing creds
+    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
         if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
 
-        clientdb.add(id, 'addon-' + appId, clientSecret, redirectURI, scope, function (error) {
+        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
             if (error) return callback(error);
 
             var env = [
@@ -268,29 +293,79 @@ function allocateOAuthCredentials(app, callback) {
     });
 }
 
-function removeOAuthCredentials(app, callback) {
+function teardownOauth(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'removeOAuthCredentials');
+    debugApp(app, 'teardownOauth');
 
-    clientdb.delByAppId('addon-' + app.id, function (error) {
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
 }
 
-function setupLdap(app, callback) {
+function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appId = app.id;
+    var id = 'cid-' + uuid.v4();
+    var scope = 'profile';
+
+    debugApp(app, 'setupSimpleAuth: id:%s', id);
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
+
+        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+            if (error) return callback(error);
+
+            var env = [
+                'SIMPLE_AUTH_SERVER=172.17.0.1',
+                'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + id
+            ];
+
+            debugApp(app, 'Setting simple auth addon config to %j', env);
+
+            appdb.setAddonConfig(appId, 'simpleauth', env, callback);
+        });
+    });
+}
+
+function teardownSimpleAuth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'teardownSimpleAuth');
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+
+        appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
+    });
+}
+
+function setupLdap(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'LDAP_SERVER=172.17.42.1',
-        'LDAP_PORT=3002',
-        'LDAP_URL=ldap://172.17.42.1:3002',
+        'LDAP_SERVER=172.17.0.1',
+        'LDAP_PORT=' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
-        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron'
+        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
+        'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
+        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
     ];
 
     debugApp(app, 'Setting up LDAP');
@@ -298,8 +373,9 @@ function setupLdap(app, callback) {
     appdb.setAddonConfig(app.id, 'ldap', env, callback);
 }
 
-function teardownLdap(app, callback) {
+function teardownLdap(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Tearing down LDAP');
@@ -307,14 +383,15 @@ function teardownLdap(app, callback) {
     appdb.unsetAddonConfig(app.id, 'ldap', callback);
 }
 
-function setupSendMail(app, callback) {
+function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
         'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=25',
-        'MAIL_SMTP_USERNAME=' + (app.location || app.id), // use app.id for bare domains
+        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
+        'MAIL_SMTP_USERNAME=' + (app.location || app.id) + '-app', // use app.id for bare domains
         'MAIL_DOMAIN=' + config.fqdn()
     ];
 
@@ -323,8 +400,9 @@ function setupSendMail(app, callback) {
     appdb.setAddonConfig(app.id, 'sendmail', env, callback);
 }
 
-function teardownSendMail(app, callback) {
+function teardownSendMail(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Tearing down sendmail');
@@ -332,8 +410,9 @@ function teardownSendMail(app, callback) {
     appdb.unsetAddonConfig(app.id, 'sendmail', callback);
 }
 
-function setupMySql(app, callback) {
+function setupMySql(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Setting up mysql');
@@ -366,7 +445,11 @@ function setupMySql(app, callback) {
     });
 }
 
-function teardownMySql(app, callback) {
+function teardownMySql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var container = docker.getContainer('mysql');
     var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
 
@@ -388,7 +471,7 @@ function teardownMySql(app, callback) {
     });
 }
 
-function backupMySql(app, callback) {
+function backupMySql(app, options, callback) {
     debugApp(app, 'Backing up mysql');
 
     callback = once(callback); // ChildProcess exit may or may not be called after error
@@ -407,10 +490,10 @@ function backupMySql(app, callback) {
     cp.stderr.pipe(process.stderr);
 }
 
-function restoreMySql(app, callback) {
+function restoreMySql(app, options, callback) {
     callback = once(callback); // ChildProcess exit may or may not be called after error
 
-    setupMySql(app, function (error) {
+    setupMySql(app, options, function (error) {
         if (error) return callback(error);
 
         debugApp(app, 'restoreMySql');
@@ -432,8 +515,9 @@ function restoreMySql(app, callback) {
     });
 }
 
-function setupPostgreSql(app, callback) {
+function setupPostgreSql(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Setting up postgresql');
@@ -466,7 +550,11 @@ function setupPostgreSql(app, callback) {
     });
 }
 
-function teardownPostgreSql(app, callback) {
+function teardownPostgreSql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
 
@@ -488,7 +576,7 @@ function teardownPostgreSql(app, callback) {
     });
 }
 
-function backupPostgreSql(app, callback) {
+function backupPostgreSql(app, options, callback) {
     debugApp(app, 'Backing up postgresql');
 
     callback = once(callback); // ChildProcess exit may or may not be called after error
@@ -507,10 +595,10 @@ function backupPostgreSql(app, callback) {
     cp.stderr.pipe(process.stderr);
 }
 
-function restorePostgreSql(app, callback) {
+function restorePostgreSql(app, options, callback) {
     callback = once(callback); // ChildProcess exit may or may not be called after error
 
-    setupPostgreSql(app, function (error) {
+    setupPostgreSql(app, options, function (error) {
         if (error) return callback(error);
 
         debugApp(app, 'restorePostgreSql');
@@ -532,8 +620,9 @@ function restorePostgreSql(app, callback) {
     });
 }
 
-function setupMongoDb(app, callback) {
+function setupMongoDb(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Setting up mongodb');
@@ -566,7 +655,11 @@ function setupMongoDb(app, callback) {
     });
 }
 
-function teardownMongoDb(app, callback) {
+function teardownMongoDb(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
 
@@ -588,7 +681,7 @@ function teardownMongoDb(app, callback) {
     });
 }
 
-function backupMongoDb(app, callback) {
+function backupMongoDb(app, options, callback) {
     debugApp(app, 'Backing up mongodb');
 
     callback = once(callback); // ChildProcess exit may or may not be called after error
@@ -607,10 +700,10 @@ function backupMongoDb(app, callback) {
     cp.stderr.pipe(process.stderr);
 }
 
-function restoreMongoDb(app, callback) {
+function restoreMongoDb(app, options, callback) {
     callback = once(callback); // ChildProcess exit may or may not be called after error
 
-    setupMongoDb(app, function (error) {
+    setupMongoDb(app, options, function (error) {
         if (error) return callback(error);
 
         debugApp(app, 'restoreMongoDb');
@@ -643,14 +736,34 @@ function forwardRedisPort(appId, callback) {
         var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
         if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
 
-        vbox.forwardFromHostToVirtualBox('redis-' + appId, redisPort);
-
         return callback(null);
     });
 }
 
+function stopAndRemoveRedis(container, callback) {
+    function ignoreError(func) {
+        return function (callback) {
+            func(function (error) {
+                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
+                callback();
+            });
+        };
+    }
+
+    // stopping redis with SIGTERM makes it commit the database to disk
+    async.series([
+        ignoreError(container.stop.bind(container, { t: 10 })),
+        ignoreError(container.wait.bind(container)),
+        ignoreError(container.remove.bind(container, { force: true, v: true }))
+    ], callback);
+}
+
 // Ensures that app's addon redis container is running. Can be called when named container already exists/running
-function setupRedis(app, callback) {
+function setupRedis(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var redisPassword = generatePassword(64, false /* memorable */);
     var redisVarsFile = path.join(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
     var redisDataDir = path.join(paths.DATA_DIR, app.id + '/redis');
@@ -663,29 +776,30 @@ function setupRedis(app, callback) {
 
     var createOptions = {
         name: 'redis-' + app.id,
-        Hostname: config.appFqdn(app.location),
+        Hostname: 'redis-' + app.location,
         Tty: true,
-        Image: 'cloudron/redis:0.3.1',
+        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
-        Volumes: {},
-        VolumesFrom: []
-    };
-
-    var isMac = os.platform() === 'darwin';
-
-    var startOptions = {
-        Binds: [
-            redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
-            redisDataDir + ':/var/lib/redis:rw'
-        ],
-        // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work
-        // On linux, export to localhost only for testing purposes and not for the app itself
-        PortBindings: {
-            '6379/tcp': [{ HostPort: '0', HostIp: isMac ? '0.0.0.0' : '127.0.0.1' }]
+        Volumes: {
+            '/tmp': {},
+            '/run': {}
         },
-        RestartPolicy: {
-            'Name': 'always',
-            'MaximumRetryCount': 0
+        VolumesFrom: [],
+        HostConfig: {
+            Binds: [
+                redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
+                redisDataDir + ':/var/lib/redis:rw'
+            ],
+            Memory: 1024 * 1024 * 75, // 100mb
+            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
+            PortBindings: {
+                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
+            },
+            ReadonlyRootfs: true,
+            RestartPolicy: {
+                'Name': 'always',
+                'MaximumRetryCount': 0
+            }
         }
     };
 
@@ -697,11 +811,11 @@ function setupRedis(app, callback) {
     ];
 
     var redisContainer = docker.getContainer(createOptions.name);
-    redisContainer.remove({ force: true, v: false }, function (ignoredError) {
+    stopAndRemoveRedis(redisContainer, function () {
         docker.createContainer(createOptions, function (error) {
             if (error && error.statusCode !== 409) return callback(error); // if not already created
 
-            redisContainer.start(startOptions, function (error) {
+            redisContainer.start(function (error) {
                 if (error && error.statusCode !== 304) return callback(error); // if not already running
 
                 appdb.setAddonConfig(app.id, 'redis', env, function (error) {
@@ -714,19 +828,21 @@ function setupRedis(app, callback) {
     });
 }
 
-function teardownRedis(app, callback) {
+function teardownRedis(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    var container = docker.getContainer('redis-' + app.id);
 
    var removeOptions = {
        force: true, // kill container if it's running
-       v: false // removes volumes associated with the container
+       v: true // removes volumes associated with the container
    };
 
    container.remove(removeOptions, function (error) {
        if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
 
-       vbox.unforwardFromHostToVirtualBox('redis-' + app.id);
-
        safe.fs.unlinkSync(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
 
         shell.sudo('teardownRedis', [ RMAPPDIR_CMD, app.id + '/redis' ], function (error, stdout, stderr) {
@@ -737,40 +853,18 @@ function teardownRedis(app, callback) {
    });
 }
 
-function allocateAccessToken(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
+function backupRedis(app, options, callback) {
+    debugApp(app, 'Backing up redis');
 
-    var token = tokendb.generateToken();
-    var expiresAt = Number.MAX_SAFE_INTEGER;    // basically never expire
-    var scopes = 'profile,users';               // TODO This should be put into the manifest and the user should know those
-    var clientId = '';                          // meaningless for apps so far
+    callback = once(callback); // ChildProcess exit may or may not be called after error
 
-   tokendb.delByIdentifier(tokendb.PREFIX_APP + app.id, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        tokendb.add(token, tokendb.PREFIX_APP + app.id, clientId, expiresAt, scopes, function (error) {
-            if (error) return callback(error);
-
-            var env = [
-                'CLOUDRON_TOKEN=' + token
-            ];
-
-            debugApp(app, 'Setting token addon config to %j', env);
-
-            appdb.setAddonConfig(appId, 'token', env, callback);
-        });
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
+        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
     });
+
+    cp.stdout.pipe(process.stdout);
+    cp.stderr.pipe(process.stderr);
 }
-
-function removeAccessToken(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delByIdentifier(tokendb.PREFIX_APP + app.id, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
-
-        appdb.unsetAddonConfig(app.id, 'token', callback);
-    });
-}
-

src/appdb.js

@@ -6,6 +6,7 @@ exports = module.exports = {
     get: get,
     getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
+    getByContainerId: getByContainerId,
     add: add,
     exists: exists,
     del: del,
@@ -35,12 +36,12 @@ exports = module.exports = {
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
-    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
     RSTATE_STOPPED: 'stopped', // app stopped by use
 
+    // run codes (keep in sync in UI)
     HEALTH_HEALTHY: 'healthy',
     HEALTH_UNHEALTHY: 'unhealthy',
     HEALTH_ERROR: 'error',
@@ -56,13 +57,9 @@ var assert = require('assert'),
     safe = require('safetydance'),
     util = require('util');
 
-var APPS_FIELDS = [ 'id', 'appStoreId', 'installationState', 'installationProgress', 'runState',
-    'health', 'containerId', 'manifestJson', 'httpPort', 'location', 'dnsRecordId',
-    'accessRestriction', 'lastBackupId', 'lastBackupConfigJson', 'oldConfigJson' ].join(',');
-
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestriction', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -94,6 +91,13 @@ function postProcess(result) {
     for (var i = 0; i < environmentVariables.length; i++) {
         result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
     }
+
+    result.oauthProxy = !!result.oauthProxy;
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
 }
 
 function get(id, callback) {
@@ -144,6 +148,22 @@ function getByHttpPort(httpPort, callback) {
     });
 }
 
+function getByContainerId(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -159,24 +179,26 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, callback) {
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
     var manifestJson = JSON.stringify(manifest);
+    var accessRestrictionJson = JSON.stringify(accessRestriction);
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestriction) VALUES (?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestriction ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -285,6 +307,9 @@ function updateWithConstraints(id, app, constraints, callback) {
         } else if (p === 'oldConfig') {
             fields.push('oldConfigJson = ?');
             values.push(JSON.stringify(app[p]));
+        } else if (p === 'accessRestriction') {
+            fields.push('accessRestrictionJson = ?');
+            values.push(JSON.stringify(app[p]));
         } else if (p !== 'portBindings') {
             fields.push(p + ' = ?');
             values.push(app[p]);
@@ -335,6 +360,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
 
     // Rules are:
     // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
     // update and configure are allowed only in installed state
 

src/apphealthmonitor.js

@@ -1,44 +1,33 @@
-#!/usr/bin/env node
-
 'use strict';
 
-require('supererror')({ splatchError: true });
-
-var appdb = require('./src/appdb.js'),
+var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    database = require('./src/database.js'),
-    DatabaseError = require('./src/databaseerror.js'),
-    debug = require('debug')('box:apphealthtask'),
-    docker = require('./src/docker.js'),
-    mailer = require('./src/mailer.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apphealthmonitor'),
+    docker = require('./docker.js').connection,
+    mailer = require('./mailer.js'),
     superagent = require('superagent'),
     util = require('util');
 
 exports = module.exports = {
-    run: run
+    start: start,
+    stop: stop
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
 var gHealthInfo = { }; // { time, emailSent }
+var gRunTimeout = null;
+var gDockerEventStream = null;
 
-function debugApp(app, args) {
+function debugApp(app) {
     assert(!app || typeof app === 'object');
 
     var prefix = app ? app.location : '(no app)';
     debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.initialize,
-        mailer.initialize
-    ], callback);
-}
-
 function setHealth(app, health, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
@@ -57,7 +46,7 @@ function setHealth(app, health, callback) {
 
         debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
 
-        mailer.appDied(app);
+        if (app.appStoreId !== '') mailer.appDied(app); // do not send mails for dev apps
         gHealthInfo[app.id].emailSent = true;
     } else {
         debugApp(app, 'waiting for sometime to update the app health');
@@ -108,7 +97,6 @@ function checkAppHealth(app, callback) {
                 debugApp(app, 'not alive : %s', error || res.status);
                 setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
             } else {
-                debugApp(app, 'alive');
                 setHealth(app, appdb.HEALTH_HEALTHY, callback);
             }
         });
@@ -121,6 +109,13 @@ function processApps(callback) {
 
         async.each(apps, checkAppHealth, function (error) {
             if (error) console.error(error);
+
+            var alive = apps
+                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
+                .map(function (a) { return a.location; }).join(', ');
+
+            debug('apps alive: [%s]', alive);
+
             callback(null);
         });
     });
@@ -130,18 +125,69 @@ function run() {
     processApps(function (error) {
         if (error) console.error(error);
 
-        setTimeout(run, HEALTHCHECK_INTERVAL);
+        gRunTimeout = setTimeout(run, HEALTHCHECK_INTERVAL);
     });
 }
 
-if (require.main === module) {
-    initialize(function (error) {
-        if (error) {
-            console.error('apphealth task exiting with error', error);
-            process.exit(1);
-        }
+/*
+    OOM can be tested using stress tool like so:
+        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        apt-get update && apt-get install stress
+        stress --vm 1 --vm-bytes 200M --vm-hang 0
+*/
+function processDockerEvents() {
+    // note that for some reason, the callback is called only on the first event
+    debug('Listening for docker events');
+    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return console.error(error);
 
-        run();
+        gDockerEventStream = stream;
+
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            var ev = JSON.parse(data);
+            debug('Container ' + ev.id + ' went OOM');
+            appdb.getByContainerId(ev.id, function (error, app) {
+                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
+                var context = JSON.stringify(ev);
+                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
+
+                debug('OOM Context: %s', context);
+
+                // do not send mails for dev apps
+                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+            });
+        });
+
+        stream.on('error', function (error) {
+            console.error('Error reading docker events', error);
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+
+        stream.on('end', function () {
+            console.error('Docker event stream ended');
+            gDockerEventStream = null; // TODO: reconnect?
+        });
     });
 }
 
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting apphealthmonitor');
+
+    processDockerEvents();
+
+    run();
+
+    callback();
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    clearTimeout(gRunTimeout);
+    gDockerEventStream.end();
+
+    callback();
+}

src/apps.js

@@ -5,6 +5,8 @@
 exports = module.exports = {
     AppsError: AppsError,
 
+    hasAccessTo: hasAccessTo,
+
     get: get,
     getBySubdomain: getBySubdomain,
     getAll: getAll,
@@ -21,7 +23,6 @@ exports = module.exports = {
     backup: backup,
     backupApp: backupApp,
 
-    getLogStream: getLogStream,
     getLogs: getLogs,
 
     start: start,
@@ -37,7 +38,8 @@ exports = module.exports = {
 
     // exported for testing
     _validateHostname: validateHostname,
-    _validatePortBindings: validatePortBindings
+    _validatePortBindings: validatePortBindings,
+    _validateAccessRestriction: validateAccessRestriction
 };
 
 var addons = require('./addons.js'),
@@ -53,11 +55,14 @@ var addons = require('./addons.js'),
     docker = require('./docker.js'),
     fs = require('fs'),
     manifestFormat = require('cloudron-manifestformat'),
+    once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
+    settings = require('./settings.js'),
     semver = require('semver'),
     shell = require('./shell.js'),
+    spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
     taskmanager = require('./taskmanager.js'),
@@ -68,6 +73,8 @@ var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
     RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
 
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
 
@@ -114,6 +121,9 @@ AppsError.BAD_STATE = 'Bad State';
 AppsError.PORT_RESERVED = 'Port Reserved';
 AppsError.PORT_CONFLICT = 'Port Conflict';
 AppsError.BILLING_REQUIRED = 'Billing Required';
+AppsError.ACCESS_DENIED = 'Access denied';
+AppsError.USER_REQUIRED = 'User required';
+AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 
 // Hostname validation comes from RFC 1123 (section 2.1)
 // Domain name validation comes from RFC 2181 (Name syntax)
@@ -140,16 +150,19 @@ function validatePortBindings(portBindings, tcpPorts) {
     // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
     // for custom tcp ports
     var RESERVED_PORTS = [
-        22, /* ssh */
         25, /* smtp */
         53, /* dns */
         80, /* http */
         443, /* https */
+        919, /* ssh */
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
         2020, /* install server */
         config.get('port'), /* app server (lo) */
         config.get('internalPort'), /* internal app server (lo) */
+        config.get('ldapPort'), /* ldap server (lo) */
+        config.get('oauthProxyPort'), /* oauth proxy server (lo) */
+        config.get('simpleAuthPort'), /* simple auth server (lo) */
         3306, /* mysql (lo) */
         8000 /* graphite (lo) */
     ];
@@ -163,7 +176,7 @@ function validatePortBindings(portBindings, tcpPorts) {
         if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
         if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
 
-        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, + portBindings[env]);
+        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, String(portBindings[env]));
     }
 
     // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
@@ -176,6 +189,18 @@ function validatePortBindings(portBindings, tcpPorts) {
     return null;
 }
 
+function validateAccessRestriction(accessRestriction) {
+    assert.strictEqual(typeof accessRestriction, 'object');
+
+    if (accessRestriction === null) return null;
+
+    if (!accessRestriction.users || !Array.isArray(accessRestriction.users)) return new Error('users array property required');
+    if (accessRestriction.users.length === 0) return new Error('users array cannot be empty');
+    if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+
+    return null;
+}
+
 function getDuplicateErrorDetails(location, portBindings, error) {
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
@@ -203,6 +228,14 @@ function getIconUrlSync(app) {
     return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
 }
 
+function hasAccessTo(app, user) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof user, 'object');
+
+    if (app.accessRestriction === null) return true;
+    return app.accessRestriction.users.some(function (e) { return e === user.id; });
+}
+
 function get(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -248,18 +281,6 @@ function getAll(callback) {
     });
 }
 
-function validateAccessRestriction(accessRestriction) {
-    // TODO: make the values below enumerations in the oauth code
-    switch (accessRestriction) {
-    case '':
-    case 'roleUser':
-    case 'roleAdmin':
-        return null;
-    default:
-        return new Error('Invalid accessRestriction');
-    }
-}
-
 function purchase(appStoreId, callback) {
     assert.strictEqual(typeof appStoreId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -272,20 +293,24 @@ function purchase(appStoreId, callback) {
     superagent.post(url).query({ token: config.token() }).end(function (error, res) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
         if (res.status === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+        if (res.status === 404) return callback(new AppsError(AppsError.NOT_FOUND));
         if (res.status !== 201 && res.status !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
 
         callback(null);
     });
 }
 
-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, callback) {
+function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, icon, cert, key, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
     assert(!icon || typeof icon === 'string');
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
     assert.strictEqual(typeof callback, 'function');
 
     var error = manifestFormat.parse(manifest);
@@ -303,6 +328,10 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     error = validateAccessRestriction(accessRestriction);
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+    // singleUser mode requires accessRestriction to contain exactly one user
+    if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
+    if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
+
     if (icon) {
         if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
 
@@ -311,15 +340,24 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
         }
     }
 
+    error = settings.validateCertificate(cert, key, config.appFqdn(location));
+    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
     debug('Will install app with id : ' + appId);
 
     purchase(appStoreId, function (error) {
         if (error) return callback(error);
 
-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, function (error) {
+        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, oauthProxy, function (error) {
             if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
+            // save cert to data/box/certs
+            if (cert && key) {
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+            }
+
             taskmanager.restartAppTask(appId);
 
             callback(null);
@@ -327,11 +365,14 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     });
 }
 
-function configure(appId, location, portBindings, accessRestriction, callback) {
+function configure(appId, location, portBindings, accessRestriction, oauthProxy, cert, key, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
     assert.strictEqual(typeof callback, 'function');
 
     var error = validateHostname(location, config.fqdn());
@@ -340,6 +381,9 @@ function configure(appId, location, portBindings, accessRestriction, callback) {
     error = validateAccessRestriction(accessRestriction);
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+    error = settings.validateCertificate(cert, key, config.appFqdn(location));
+    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -347,15 +391,23 @@ function configure(appId, location, portBindings, accessRestriction, callback) {
         error = validatePortBindings(portBindings, app.manifest.tcpPorts);
         if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+        // save cert to data/box/certs
+        if (cert && key) {
+            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        }
+
         var values = {
             location: location.toLowerCase(),
             accessRestriction: accessRestriction,
+            oauthProxy: oauthProxy,
             portBindings: portBindings,
 
             oldConfig: {
                 location: app.location,
                 accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings
+                portBindings: app.portBindings,
+                oauthProxy: app.oauthProxy
             }
         };
 
@@ -409,7 +461,9 @@ function update(appId, force, manifest, portBindings, icon, callback) {
             portBindings: portBindings,
             oldConfig: {
                 manifest: app.manifest,
-                portBindings: app.portBindings
+                portBindings: app.portBindings,
+                accessRestriction: app.accessRestriction,
+                oauthProxy: app.oauthProxy
             }
         };
 
@@ -425,58 +479,50 @@ function update(appId, force, manifest, portBindings, icon, callback) {
     });
 }
 
-function getLogStream(appId, fromLine, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof fromLine, 'number'); // behaves like tail -n
-    assert.strictEqual(typeof callback, 'function');
+function appLogFilter(app) {
+    var names = [ app.id ].concat(addons.getContainerNamesSync(app, app.manifest.addons));
 
-    debug('Getting logs for %s', appId);
-    appdb.get(appId, function (error, app) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
-
-        var container = docker.getContainer(app.containerId);
-        var tail = fromLine < 0 ? -fromLine : 'all';
-
-        // note: cannot access docker file directly because it needs root access
-        container.logs({ stdout: true, stderr: true, follow: true, timestamps: true, tail: tail }, function (error, logStream) {
-            if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            var lineCount = 0;
-            var skipLinesStream = split(function mapper(line) {
-                if (++lineCount < fromLine) return undefined;
-                var timestamp = line.substr(0, line.indexOf(' ')); // sometimes this has square brackets around it
-                return JSON.stringify({ lineNumber: lineCount, timestamp: timestamp.replace(/[[\]]/g,''), log: line.substr(timestamp.length + 1) });
-            });
-            skipLinesStream.close = logStream.req.abort;
-            logStream.pipe(skipLinesStream);
-            return callback(null, skipLinesStream);
-        });
-    });
+    return names.map(function (name) { return 'CONTAINER_NAME=' + name; });
 }
 
-function getLogs(appId, callback) {
+function getLogs(appId, lines, follow, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lines, 'number');
+    assert.strictEqual(typeof follow, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Getting logs for %s', appId);
+
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
         if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
 
-        var container = docker.getContainer(app.containerId);
-        // note: cannot access docker file directly because it needs root access
-        container.logs({ stdout: true, stderr: true, follow: false, timestamps: true, tail: 'all' }, function (error, logStream) {
-            if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        var args = [ '--output=json', '--no-pager', '--lines=' + lines ];
+        if (follow) args.push('--follow');
+        args = args.concat(appLogFilter(app));
 
-            return callback(null, logStream);
+        var cp = spawn('/bin/journalctl', args);
+
+        var transformStream = split(function mapper(line) {
+            var obj = safe.JSON.parse(line);
+            if (!obj) return undefined;
+
+            var source = obj.CONTAINER_NAME.slice(app.id.length + 1);
+            return JSON.stringify({
+                realtimeTimestamp: obj.__REALTIME_TIMESTAMP,
+                monotonicTimestamp: obj.__MONOTONIC_TIMESTAMP,
+                message: obj.MESSAGE,
+                source: source || 'main'
+            }) + '\n';
         });
+
+        transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process
+
+        cp.stdout.pipe(transformStream);
+
+        return callback(null, transformStream);
     });
 }
 
@@ -490,28 +536,31 @@ function restore(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var restoreConfig = app.lastBackupConfig;
-        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+        // restore without a backup is the same as re-install
+        var restoreConfig = app.lastBackupConfig, values = { };
+        if (restoreConfig) {
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
-        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
-        error = checkManifestConstraints(restoreConfig.manifest);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
+            if (error) return callback(error);
 
-        error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
-        if (error) return callback(error);
+            // ## should probably query new location, access restriction from user
+            values = {
+                manifest: restoreConfig.manifest,
+                portBindings: restoreConfig.portBindings,
 
-        // ## should probably query new location, access restriction from user
-        var values = {
-            manifest: restoreConfig.manifest,
-            portBindings: restoreConfig.portBindings,
-
-            oldConfig: {
-                location: app.location,
-                accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings,
-                manifest: app.manifest
-            }
-        };
+                oldConfig: {
+                    location: app.location,
+                    accessRestriction: app.accessRestriction,
+                    oauthProxy: app.oauthProxy,
+                    portBindings: app.portBindings,
+                    manifest: app.manifest
+                }
+            };
+        }
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -573,6 +622,8 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
         return new Error('Box version exceeds Apps maxBoxVersion');
     }
@@ -596,31 +647,42 @@ function exec(appId, options, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var container = docker.getContainer(app.containerId);
-
-       var execOptions = {
+       var createOptions = {
             AttachStdin: true,
             AttachStdout: true,
             AttachStderr: true,
-            Tty: true,
-            Cmd: cmd
+            OpenStdin: true,
+            StdinOnce: false,
+            Tty: true
         };
 
-        container.exec(execOptions, function (error, exec) {
+        docker.createSubcontainer(app, app.id + '-exec-' + Date.now(), cmd, createOptions, function (error, container) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-            var startOptions = {
-                Detach: false,
-                Tty: true,
-                stdin: true // this is a dockerode option that enabled openStdin in the modem
-            };
-            exec.start(startOptions, function(error, stream) {
+
+            container.attach({ stream: true, stdin: true, stdout: true, stderr: true }, function (error, stream) {
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                if (options.rows && options.columns) {
-                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-                }
+                docker.startContainer(container.id, function (error) {
+                    if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                return callback(null, stream);
+                    if (options.rows && options.columns) {
+                        container.resize({ h: options.rows, w: options.columns }, NOOP_CALLBACK);
+                    }
+
+                    var deleteContainer = once(docker.deleteContainer.bind(null, container.id, NOOP_CALLBACK));
+
+                    container.wait(function (error) {
+                        if (error) debug('Error waiting on container', error);
+
+                        debug('exec: container finished', container.id);
+
+                        deleteContainer();
+                    });
+
+                    stream.close = deleteContainer;
+
+                    callback(null, stream);
+                });
             });
         });
     });
@@ -645,27 +707,38 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     assert.strictEqual(typeof callback, 'function');
 
     function canAutoupdateApp(app, newManifest) {
-        // TODO: maybe check the description as well?
-        if (!newManifest.tcpPorts && !app.portBindings) return true;
-        if (!newManifest.tcpPorts || !app.portBindings) return false;
+        var tcpPorts = newManifest.tcpPorts || { };
+        var portBindings = app.portBindings; // this is never null
 
-        for (var env in newManifest.tcpPorts) {
-            if (!(env in app.portBindings)) return false;
-       }
+        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
+        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
+        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
 
-        return true;
+        for (var env in tcpPorts) {
+            if (!(env in portBindings)) return new Error(env + ' is required from user');
+        }
+
+        // it's fine if one or more keys got removed
+        return null;
     }
 
     if (!updateInfo) return callback(null);
 
     async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
         get(appId, function (error, app) {
-            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+            if (error) {
+                debug('Cannot autoupdate app %s : %s', appId, error.message);
+                return iteratorDone();
+           }
+
+            error = canAutoupdateApp(app, updateInfo[appId].manifest);
+            if (error) {
+                debug('app %s requires manual update. %s', appId, error.message);
                 return iteratorDone();
             }
 
-           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
-                if (error) debug('Error initiating autoupdate of %s', appId);
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+                if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);
 
                 iteratorDone(null);
             });
@@ -673,34 +746,36 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     }, callback);
 }
 
-function backupApp(app, addonsToBackup, callback) {
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldBackup(app, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    backups.copyLastBackup(app, function (error, newBackupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewBackup(app, addonsToBackup, callback) {
     assert.strictEqual(typeof app, 'object');
     assert(!addonsToBackup || typeof addonsToBackup, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    function canBackupApp(app) {
-        // only backup apps that are installed or pending configure. Rest of them are in some
-        // state not good for consistent backup (i.e addons may not have been setup completely)
-        return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
-                app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
-                app.installationState === appdb.ISTATE_PENDING_BACKUP ||
-                app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
-    }
-
-    if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy'));
-
-    var appConfig = {
-        manifest: app.manifest,
-        location: app.location,
-        portBindings: app.portBindings,
-        accessRestriction: app.accessRestriction
-    };
-
-    if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
-        return callback(safe.error);
-    }
-
-    backups.getBackupUrl(app, null, function (error, result) {
+    backups.getBackupUrl(app, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -709,18 +784,55 @@ function backupApp(app, addonsToBackup, callback) {
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
             addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            debugApp(app, 'backupApp: successful id:%s', result.id);
+            callback(null, result.id);
+        });
+    });
+}
 
-            setRestorePoint(app.id, result.id, appConfig, function (error) {
-                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+function backupApp(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-                return callback(null, result.id);
-            });
+    var appConfig = null, backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        appConfig = app.lastBackupConfig;
+        backupFunction = reuseOldBackup.bind(null, app);
+    } else {
+        appConfig = {
+            manifest: app.manifest,
+            location: app.location,
+            portBindings: app.portBindings,
+            accessRestriction: app.accessRestriction,
+            oauthProxy: app.oauthProxy
+        };
+        backupFunction = createNewBackup.bind(null, app, addonsToBackup);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, appConfig, function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            return callback(null, backupId);
         });
     });
 }
@@ -756,11 +868,10 @@ function restoreApp(app, addonsToRestore, callback) {
 
         debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
 
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             addons.restoreAddons(app, addonsToRestore, callback);
         });
     });
 }
-

src/apptask.js

@@ -25,6 +25,12 @@ exports = module.exports = {
 
 require('supererror')({ splatchError: true });
 
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
 var addons = require('./addons.js'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
@@ -41,16 +47,16 @@ var addons = require('./addons.js'),
     hat = require('hat'),
     manifestFormat = require('cloudron-manifestformat'),
     net = require('net'),
-    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    SubdomainError = require('./subdomains.js').SubdomainError,
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
     uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
     _ = require('underscore');
 
 var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
@@ -92,8 +98,21 @@ function configureNginx(app, callback) {
         if (error) return callback(error);
 
         var sourceDir = path.resolve(__dirname, '..');
-        var endpoint = app.accessRestriction ? 'oauthproxy' : 'app';
-        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, { sourceDir: sourceDir, adminOrigin: config.adminOrigin(), vhost: config.appFqdn(app.location), port: freePort, endpoint: endpoint });
+        var endpoint = app.oauthProxy ? 'oauthproxy' : 'app';
+        var vhost = config.appFqdn(app.location);
+        var certFilePath = safe.fs.statSync(path.join(paths.APP_CERTS_DIR, vhost + '.cert')) ? path.join(paths.APP_CERTS_DIR, vhost + '.cert') : 'cert/host.cert';
+        var keyFilePath = safe.fs.statSync(path.join(paths.APP_CERTS_DIR, vhost + '.key')) ? path.join(paths.APP_CERTS_DIR, vhost + '.key') : 'cert/host.key';
+
+        var data = {
+            sourceDir: sourceDir,
+            adminOrigin: config.adminOrigin(),
+            vhost: vhost,
+            port: freePort,
+            endpoint: endpoint,
+            certFilePath: certFilePath,
+            keyFilePath: keyFilePath
+        };
+        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
 
         var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
         debugApp(app, 'writing config to %s', nginxConfigFilename);
@@ -107,8 +126,6 @@ function configureNginx(app, callback) {
             exports._reloadNginx,
             updateApp.bind(null, app, { httpPort: freePort })
         ], callback);
-
-        vbox.forwardFromHostToVirtualBox(app.id + '-http', freePort);
     });
 }
 
@@ -120,143 +137,27 @@ function unconfigureNginx(app, callback) {
     }
 
     exports._reloadNginx(callback);
-
-    vbox.unforwardFromHostToVirtualBox(app.id + '-http');
-}
-
-function downloadImage(app, callback) {
-    debugApp(app, 'downloadImage %s', app.manifest.dockerImage);
-
-    docker.pull(app.manifest.dockerImage, function (err, stream) {
-        if (err) return callback(new Error('Error connecting to docker'));
-
-        // https://github.com/dotcloud/docker/issues/1074 says each status message
-        // is emitted as a chunk
-        stream.on('data', function (chunk) {
-            var data = safe.JSON.parse(chunk) || { };
-            debugApp(app, 'downloadImage data: %j', data);
-
-            // The information here is useless because this is per layer as opposed to per image
-            if (data.status) {
-                debugApp(app, 'progress: %s', data.status); // progressDetail { current, total }
-            } else if (data.error) {
-                debugApp(app, 'error detail: %s', data.errorDetail.message);
-            }
-        });
-
-        stream.on('end', function () {
-            debugApp(app, 'download image successfully');
-
-            var image = docker.getImage(app.manifest.dockerImage);
-
-            image.inspect(function (err, data) {
-                if (err) {
-                    return callback(new Error('Error inspecting image:' + err.message));
-                }
-
-                if (!data || !data.Config) {
-                    return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
-                }
-
-                if (!data.Config.Entrypoint && !data.Config.Cmd) {
-                    return callback(new Error('Only images with entry point are allowed'));
-                }
-
-                debugApp(app, 'This image exposes ports: %j', data.Config.ExposedPorts);
-                return callback(null);
-            });
-        });
-    });
 }
 
 function createContainer(app, callback) {
-    appdb.getPortBindings(app.id, function (error, portBindings) {
-        if (error) return callback(error);
+    assert(!app.containerId); // otherwise, it will trigger volumeFrom
 
-        var manifest = app.manifest;
-        var exposedPorts = {};
-        var env = [];
+    debugApp(app, 'creating container');
 
-        // docker portBindings requires ports to be exposed
-        exposedPorts[manifest.httpPort + '/tcp'] = {};
+    docker.createContainer(app, function (error, container) {
+        if (error) return callback(new Error('Error creating container: ' + error));
 
-        for (var e in portBindings) {
-            var hostPort = portBindings[e];
-            var containerPort = manifest.tcpPorts[e].containerPort || hostPort;
-            exposedPorts[containerPort + '/tcp'] = {};
-
-            env.push(e + '=' + hostPort);
-        }
-
-        env.push('CLOUDRON=1');
-        env.push('WEBADMIN_ORIGIN' + '=' + config.adminOrigin());
-        env.push('API_ORIGIN' + '=' + config.adminOrigin());
-
-        addons.getEnvironment(app, function (error, addonEnv) {
-            if (error) return callback(new Error('Error getting addon env: ' + error));
-
-            var containerOptions = {
-                name: app.id,
-                Hostname: config.appFqdn(app.location),
-                Tty: true,
-                Image: app.manifest.dockerImage,
-                Cmd: null,
-                Env: env.concat(addonEnv),
-                ExposedPorts: exposedPorts
-            };
-
-            debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
-
-            docker.createContainer(containerOptions, function (error, container) {
-                if (error) return callback(new Error('Error creating container: ' + error));
-
-                updateApp(app, { containerId: container.id }, callback);
-            });
-        });
+        updateApp(app, { containerId: container.id }, callback);
     });
 }
 
-function deleteContainer(app, callback) {
-    if (app.containerId === null) return callback(null);
+function deleteContainers(app, callback) {
+    debugApp(app, 'deleting containers');
 
-    var container = docker.getContainer(app.containerId);
+    docker.deleteContainers(app.id, function (error) {
+        if (error) return callback(new Error('Error deleting container: ' + error));
 
-    var removeOptions = {
-        force: true, // kill container if it's running
-        v: false // removes volumes associated with the container
-    };
-
-    container.remove(removeOptions, function (error) {
-        if (error && error.statusCode === 404) return updateApp(app, { containerId: null }, callback);
-
-        if (error) debugApp(app, 'Error removing container', error);
-        callback(error);
-    });
-}
-
-function deleteImage(app, manifest, callback) {
-    var dockerImage = manifest ? manifest.dockerImage : null;
-    if (!dockerImage) return callback(null);
-
-    docker.getImage(dockerImage).inspect(function (error, result) {
-        if (error && error.statusCode === 404) return callback(null);
-
-        if (error) return callback(error);
-
-        var removeOptions = {
-            force: true,
-            noprune: false
-        };
-
-        // delete image by id because docker pull pulls down all the tags and this is the only way to delete all tags
-        docker.getImage(result.Id).remove(removeOptions, function (error) {
-            if (error && error.statusCode === 404) return callback(null);
-            if (error && error.statusCode === 409) return callback(null); // another container using the image
-
-            if (error) debugApp(app, 'Error removing image', error);
-
-            callback(error);
-        });
+        updateApp(app, { containerId: null }, callback);
     });
 }
 
@@ -272,22 +173,21 @@ function allocateOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!app.accessRestriction) return callback(null);
+    if (!app.oauthProxy) return callback(null);
 
-    var appId = 'proxy-' + app.id;
-    var id = 'cid-proxy-' + uuid.v4();
+    var id = 'cid-' + uuid.v4();
     var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
-    var scope = 'profile,' + app.accessRestriction;
+    var scope = 'profile';
 
-    clientdb.add(id, appId, clientSecret, redirectURI, scope, callback);
+    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
 }
 
 function removeOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppId('proxy-' + app.id, function (error) {
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) {
             debugApp(app, 'Error removing OAuth client id', error);
             return callback(error);
@@ -312,76 +212,6 @@ function removeCollectdProfile(app, callback) {
     });
 }
 
-function startContainer(app, callback) {
-    appdb.getPortBindings(app.id, function (error, portBindings) {
-        if (error) return callback(error);
-
-        var manifest = app.manifest;
-
-        var dockerPortBindings = { };
-        var isMac = os.platform() === 'darwin';
-
-        // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work
-        dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: isMac ? '0.0.0.0' : '127.0.0.1', HostPort: app.httpPort + '' } ];
-
-        for (var env in portBindings) {
-            var hostPort = portBindings[env];
-            var containerPort = manifest.tcpPorts[env].containerPort || hostPort;
-            dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
-            vbox.forwardFromHostToVirtualBox(app.id + '-tcp' + containerPort, hostPort);
-        }
-
-        var startOptions = {
-            Binds: addons.getBindsSync(app, app.manifest.addons),
-            PortBindings: dockerPortBindings,
-            PublishAllPorts: false,
-            Links: addons.getLinksSync(app, app.manifest.addons),
-            RestartPolicy: {
-                "Name": "always",
-                "MaximumRetryCount": 0
-            },
-            CpuShares: 512 // relative to 1024 for system processes
-        };
-
-        var container = docker.getContainer(app.containerId);
-        debugApp(app, 'Starting container %s with options: %j', container.id, JSON.stringify(startOptions));
-
-        container.start(startOptions, function (error, data) {
-            if (error && error.statusCode !== 304) return callback(new Error('Error starting container:' + error));
-
-            return callback(null);
-        });
-    });
-}
-
-function stopContainer(app, callback) {
-    var container = docker.getContainer(app.containerId);
-    debugApp(app, 'Stopping container %s', container.id);
-
-    var options = {
-        t: 10 // wait for 10 seconds before killing it
-    };
-
-    container.stop(options, function (error) {
-        if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error stopping container:' + error));
-
-        var tcpPorts = safe.query(app, 'manifest.tcpPorts', { });
-        for (var containerPort in tcpPorts) {
-            vbox.unforwardFromHostToVirtualBox(app.id + '-tcp' + containerPort);
-        }
-
-        debugApp(app, 'Waiting for container ' + container.id);
-
-        container.wait(function (error, data) {
-            if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error waiting on container:' + error));
-
-            debugApp(app, 'Container stopped with status code [%s]', data ? String(data.StatusCode) : '');
-
-            return callback(null);
-        });
-    });
-}
-
 function verifyManifest(app, callback) {
     debugApp(app, 'Verifying manifest');
 
@@ -414,49 +244,43 @@ function downloadIcon(app, callback) {
 }
 
 function registerSubdomain(app, callback) {
-    debugApp(app, 'Registering subdomain location [%s]', app.location);
-
     // even though the bare domain is already registered in the appstore, we still
     // need to register it so that we have a dnsRecordId to wait for it to complete
-    var record = { subdomain: app.location, type: 'A', value: sysinfo.getIp() };
+    async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Registering subdomain location [%s]', app.location);
 
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: [ record ] })
-        .end(function (error, res) {
-            if (error) return callback(error);
+        subdomains.add(app.location, 'A', [ sysinfo.getIp() ], function (error, changeId) {
+            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
 
-            debugApp(app, 'Registered subdomain status: %s', res.status);
-
-            if (res.status === 409) return callback(null); // already registered
-            if (res.status !== 201) return callback(new Error(util.format('Subdomain Registration failed. %s %j', res.status, res.body)));
-
-            updateApp(app, { dnsRecordId: res.body.ids[0] }, callback);
+            retryCallback(null, error || changeId);
         });
+    }, function (error, result) {
+        if (error || result instanceof Error) return callback(error || result);
+
+        updateApp(app, { dnsRecordId: result }, callback);
+    });
 }
 
-function unregisterSubdomain(app, callback) {
-    debugApp(app, 'Unregistering subdomain: dnsRecordId=%s', app.dnsRecordId);
-
-    if (!app.dnsRecordId) return callback(null);
-
+function unregisterSubdomain(app, location, callback) {
     // do not unregister bare domain because we show a error/cloudron info page there
-    if (app.location === '') return updateApp(app, { dnsRecordId: null }, callback);
+    if (location === '') {
+        debugApp(app, 'Skip unregister of empty subdomain');
+        return callback(null);
+    }
 
-    superagent
-        .del(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId)
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) {
-                debugApp(app, 'Error making request: %s', error);
-            } else if (res.status !== 204) {
-                debugApp(app, 'Error unregistering subdomain:', res.status, res.body);
-            }
+    async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Unregistering subdomain: %s', location);
 
-            updateApp(app, { dnsRecordId: null }, callback);
+        subdomains.remove(location, 'A', [ sysinfo.getIp() ], function (error) {
+            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+            retryCallback(null, error);
         });
+    }, function (error, result) {
+        if (error || result instanceof Error) return callback(error || result);
+
+        updateApp(app, { dnsRecordId: null }, callback);
+    });
 }
 
 function removeIcon(app, callback) {
@@ -477,26 +301,20 @@ function waitForDnsPropagation(app, callback) {
         setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
     }
 
-    superagent
-        .get(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId + '/status')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+    subdomains.status(app.dnsRecordId, function (error, result) {
+        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
 
-            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, res.status);
+        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-            if (res.status !== 200) return retry(new Error(util.format('Error getting record status: %s %j', res.status, res.body)));
+        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-            if (res.body.status !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, res.body.status)));
-
-            callback(null);
-        });
+        callback(null);
+    });
 }
 
 // updates the app object and the database
 function updateApp(app, values, callback) {
-    debugApp(app, 'installationState: %s progress: %s', app.installationState, app.installationProgress);
+    debugApp(app, 'updating app with values: %j', values);
 
     appdb.update(app.id, values, function (error) {
         if (error) return callback(error);
@@ -527,12 +345,12 @@ function install(app, callback) {
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
         deleteVolume.bind(null, app),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
         removeOAuthProxyCredentials.bind(null, app),
-        removeIcon.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
         unconfigureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
@@ -548,7 +366,7 @@ function install(app, callback) {
         registerSubdomain.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '40, Downloading image' }),
-        downloadImage.bind(null, app),
+        docker.downloadImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '50, Creating volume' }),
         createVolume.bind(null, app),
@@ -613,11 +431,15 @@ function restore(app, callback) {
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
          // oldConfig can be null during upgrades
         addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
         deleteVolume.bind(null, app),
-        deleteImage.bind(null, app, app.manifest),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             docker.deleteImage(app.oldConfig.manifest, done);
+        },
         removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -635,7 +457,7 @@ function restore(app, callback) {
         registerSubdomain.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '60, Downloading image' }),
-        downloadImage.bind(null, app),
+        docker.downloadImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '65, Creating volume' }),
         createVolume.bind(null, app),
@@ -671,16 +493,15 @@ function restore(app, callback) {
 
 // note that configure is called after an infra update as well
 function configure(app, callback) {
-    var locationChanged = app.oldConfig ? app.oldConfig.location !== app.location : true;
-
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
         function (next) {
-            if (!locationChanged) return next();
-            unregisterSubdomain(app, next);
+            // oldConfig can be null during an infra update
+            if (!app.oldConfig || app.oldConfig.location === app.location) return next();
+            unregisterSubdomain(app, app.oldConfig.location, next);
         },
         removeOAuthProxyCredentials.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -691,14 +512,8 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
         allocateOAuthProxyCredentials.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
-                registerSubdomain.bind(null, app)
-            ], next);
-        },
+        updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
+        registerSubdomain.bind(null, app),
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         updateApp.bind(null, app, { installationProgress: '50, Setting up addons' }),
@@ -712,14 +527,8 @@ function configure(app, callback) {
 
         runApp.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
-                exports._waitForDnsPropagation.bind(null, app)
-            ], next);
-        },
+        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
+        exports._waitForDnsPropagation.bind(null, app),
 
         // done!
         function (callback) {
@@ -751,9 +560,13 @@ function update(app, callback) {
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
         addons.teardownAddons.bind(null, app, unusedAddons),
-        deleteImage.bind(null, app, app.manifest), // delete image even if did not change (see df158b111f)
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             docker.deleteImage(app.oldConfig.manifest, done);
+        },
         // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
 
         function (next) {
@@ -769,7 +582,7 @@ function update(app, callback) {
         downloadIcon.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '45, Downloading image' }),
-        downloadImage.bind(null, app),
+        docker.downloadImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '70, Updating addons' }),
         addons.setupAddons.bind(null, app, app.manifest.addons),
@@ -807,7 +620,7 @@ function uninstall(app, callback) {
         stopApp.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '20, Deleting container' }),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '30, Teardown addons' }),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
@@ -816,10 +629,10 @@ function uninstall(app, callback) {
         deleteVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '50, Deleting image' }),
-        deleteImage.bind(null, app, app.manifest),
+        docker.deleteImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
 
         updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
         removeOAuthProxyCredentials.bind(null, app),
@@ -836,18 +649,15 @@ function uninstall(app, callback) {
 }
 
 function runApp(app, callback) {
-    startContainer(app, function (error) {
-        if (error) {
-            debugApp(app, 'Error starting container : %s', error);
-            return updateApp(app, { runState: appdb.RSTATE_ERROR }, callback);
-        }
+    docker.startContainer(app.containerId, function (error) {
+        if (error) return callback(error);
 
         updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
     });
 }
 
 function stopApp(app, callback) {
-    stopContainer(app, function (error) {
+    docker.stopContainers(app.id, function (error) {
         if (error) return callback(error);
 
         updateApp(app, { runState: appdb.RSTATE_STOPPED }, callback);
@@ -904,7 +714,7 @@ if (require.main === module) {
         if (error) throw error;
 
         startTask(process.argv[2], function (error) {
-            if (error) console.error(error);
+            if (error) debug('Apptask completed with error', error);
 
             debug('Apptask completed for %s', process.argv[2]);
             // https://nodejs.org/api/process.html are exit codes used by node. apps.js uses the value below

src/backups.js

@@ -6,13 +6,17 @@ exports = module.exports = {
     getAllPaged: getAllPaged,
 
     getBackupUrl: getBackupUrl,
-    getRestoreUrl: getRestoreUrl
+    getRestoreUrl: getRestoreUrl,
+
+    copyLastBackup: copyLastBackup
 };
 
 var assert = require('assert'),
+    caas = require('./storage/caas.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
-    superagent = require('superagent'),
+    s3 = require('./storage/s3.js'),
+    settings = require('./settings.js'),
     util = require('util');
 
 function BackupsError(reason, errorOrMessage) {
@@ -36,60 +40,103 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.MISSING_CREDENTIALS = 'missing credentials';
+
+// choose which storage backend we use for test purpose we use s3
+function api(provider) {
+    switch (provider) {
+        case 'caas': return caas;
+        case 's3': return s3;
+        default: return null;
+    }
+}
 
 function getAllPaged(page, perPage, callback) {
     assert.strictEqual(typeof page, 'number');
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    superagent.get(url).query({ token: config.token() }).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !util.isArray(result.body.backups)) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
-        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
-        return callback(null, result.body.backups);
+            return callback(null, backups); // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first
+        });
     });
 }
 
-function getBackupUrl(app, appBackupIds, callback) {
+function getBackupUrl(app, callback) {
     assert(!app || typeof app === 'object');
-    assert(!appBackupIds || util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+    var filename = '';
+    if (app) {
+        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    } else {
+        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
+    }
 
-    var data = {
-        boxVersion: config.version(),
-        appId: app ? app.id : null,
-        appVersion: app ? app.manifest.version : null,
-        appBackupIds: appBackupIds
-    };
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+            if (error) return callback(error);
 
-        return callback(null, result.body);
+            var obj = {
+                id: filename,
+                url: result.url,
+                sessionToken: result.sessionToken,
+                backupKey: backupConfig.key
+            };
+
+            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+            callback(null, obj);
+        });
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
+            if (error) return callback(error);
 
-        return callback(null, result.body);
+            var obj = {
+                id: backupId,
+                url: result.url,
+                sessionToken: result.sessionToken,
+                backupKey: backupConfig.key
+            };
+
+            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+            callback(null, obj);
+        });
     });
 }
 
+function copyLastBackup(app, callback) {
+    assert(app && typeof app === 'object');
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
+    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilename, function (error) {
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+            return callback(null, toFilename);
+        });
+    });
+}

src/clientdb.js

@@ -8,19 +8,27 @@ exports = module.exports = {
     getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
-    update: update,
     getByAppId: getByAppId,
-    delByAppId: delByAppId,
+    getByAppIdAndType: getByAppIdAndType,
 
-    _clear: clear
+    delByAppId: delByAppId,
+    delByAppIdAndType: delByAppIdAndType,
+
+    _clear: clear,
+
+    TYPE_EXTERNAL: 'external',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy',
+    TYPE_ADMIN: 'admin'
 };
 
 var assert = require('assert'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror.js');
 
-var CLIENTS_FIELDS = [ 'id', 'appId', 'clientSecret', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
+var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
+var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
 
 function get(id, callback) {
     assert.strictEqual(typeof id, 'string');
@@ -67,37 +75,33 @@ function getByAppId(appId, callback) {
     });
 }
 
-function add(id, appId, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
+function getByAppIdAndType(appId, type, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
+    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var data = [ id, appId, clientSecret, redirectURI, scope ];
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-    database.query('INSERT INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        callback(null);
+        return callback(null, result[0]);
     });
 }
 
-function update(id, appId, clientSecret, redirectURI, scope, callback) {
+function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof clientSecret, 'string');
     assert.strictEqual(typeof redirectURI, 'string');
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var data = [ appId, clientSecret, redirectURI, scope, id ];
+    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
 
-    database.query('UPDATE clients SET appId = ?, clientSecret = ?, redirectURI = ?, scope = ? WHERE id = ?', data, function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
     });
@@ -127,6 +131,19 @@ function delByAppId(appId, callback) {
     });
 }
 
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 

src/clients.js

@@ -5,7 +5,6 @@ exports = module.exports = {
 
     add: add,
     get: get,
-    update: update,
     del: del,
     getAllWithDetailsByUserId: getAllWithDetailsByUserId,
     getClientTokensByUserId: getClientTokensByUserId,
@@ -43,6 +42,7 @@ function ClientsError(reason, errorOrMessage) {
 }
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
+ClientsError.INVALID_CLIENT = 'Invalid client';
 
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
@@ -55,8 +55,9 @@ function validateScope(scope) {
     return null;
 }
 
-function add(appIdentifier, redirectURI, scope, callback) {
-    assert.strictEqual(typeof appIdentifier, 'string');
+function add(appId, type, redirectURI, scope, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof redirectURI, 'string');
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -67,12 +68,13 @@ function add(appIdentifier, redirectURI, scope, callback) {
     var id = 'cid-' + uuid.v4();
     var clientSecret = hat(256);
 
-    clientdb.add(id, appIdentifier, clientSecret, redirectURI, scope, function (error) {
+    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
         if (error) return callback(error);
 
         var client = {
             id: id,
-            appId: appIdentifier,
+            appId: appId,
+            type: type,
             clientSecret: clientSecret,
             redirectURI: redirectURI,
             scope: scope
@@ -92,23 +94,6 @@ function get(id, callback) {
     });
 }
 
-// we only allow appIdentifier and redirectURI to be updated
-function update(id, appIdentifier, redirectURI, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appIdentifier, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.get(id, function (error, result) {
-        if (error) return callback(error);
-
-        clientdb.update(id, appIdentifier, result.clientSecret, redirectURI, result.scope, function (error, result) {
-            if (error) return callback(error);
-            callback(null, result);
-        });
-    });
-}
-
 function del(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -127,54 +112,29 @@ function getAllWithDetailsByUserId(userId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
         if (error) return callback(error);
 
-        // We have several types of records here
-        //   1) webadmin has an app id of 'webadmin'
-        //   2) oauth proxy records are always the app id prefixed with 'proxy-'
-        //   3) addon oauth records for apps prefixed with 'addon-'
-        //   4) external app records prefixed with 'external-'
-        //   5) normal apps on the cloudron without a prefix
-
         var tmp = [];
         async.each(results, function (record, callback) {
-            if (record.appId === constants.ADMIN_CLIENT_ID) {
+            if (record.type === clientdb.TYPE_ADMIN) {
                 record.name = constants.ADMIN_NAME;
                 record.location = constants.ADMIN_LOCATION;
-                record.type = 'webadmin';
-
-                tmp.push(record);
-
-                return callback(null);
-            } else if (record.appId === constants.TEST_CLIENT_ID) {
-                record.name = constants.TEST_NAME;
-                record.location = constants.TEST_LOCATION;
-                record.type = 'test';
 
                 tmp.push(record);
 
                 return callback(null);
             }
 
-            var appId = record.appId;
-            var type = 'app';
-
-            // Handle our different types of oauth clients
-            if (record.appId.indexOf('addon-') === 0) {
-                appId = record.appId.slice('addon-'.length);
-                type = 'addon';
-            } else if (record.appId.indexOf('proxy-') === 0) {
-                appId = record.appId.slice('proxy-'.length);
-                type = 'proxy';
-            }
-
-            appdb.get(appId, function (error, result) {
+            appdb.get(record.appId, function (error, result) {
                 if (error) {
                     console.error('Failed to get app details for oauth client', result, error);
                     return callback(null);  // ignore error so we continue listing clients
                 }
 
-                record.name = result.manifest.title + (record.appId.indexOf('proxy-') === 0 ? 'OAuth Proxy' : '');
+                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
+                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+
                 record.location = result.location;
-                record.type = type;
 
                 tmp.push(record);
 

src/cloudron.js

@@ -11,15 +11,21 @@ exports = module.exports = {
     getConfig: getConfig,
     getStatus: getStatus,
 
-    setCertificate: setCertificate,
-
     sendHeartbeat: sendHeartbeat,
 
     update: update,
     reboot: reboot,
     migrate: migrate,
     backup: backup,
-    ensureBackup: ensureBackup};
+    ensureBackup: ensureBackup,
+
+    isConfiguredSync: isConfiguredSync,
+
+    events: new (require('events').EventEmitter)(),
+
+    EVENT_ACTIVATED: 'activated',
+    EVENT_CONFIGURED: 'configured'
+};
 
 var apps = require('./apps.js'),
     AppsError = require('./apps.js').AppsError,
@@ -27,6 +33,7 @@ var apps = require('./apps.js'),
     async = require('async'),
     backups = require('./backups.js'),
     BackupsError = require('./backups.js').BackupsError,
+    bytes = require('bytes'),
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
     debug = require('debug')('box:cloudron'),
@@ -37,8 +44,8 @@ var apps = require('./apps.js'),
     progress = require('./progress.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
-    SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
@@ -46,16 +53,19 @@ var apps = require('./apps.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
 
-var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
-    REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
+var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
     BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
 
-var gAddMailDnsRecordsTimerId = null,
-    gCloudronDetails = null;            // cached cloudron details like region,size...
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+var gUpdatingDns = false,                // flag for dns update reentrancy
+    gCloudronDetails = null,             // cached cloudron details like region,size...
+    gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
 
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
@@ -73,7 +83,6 @@ function ignoreError(func) {
     };
 }
 
-
 function CloudronError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -107,25 +116,61 @@ CloudronError.NOT_FOUND = 'Not found';
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (process.env.BOX_ENV !== 'test') {
-        addMailDnsRecords();
-    }
+    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
 
-    // Send heartbeat once we are up and running, this speeds up the Cloudron creation, as otherwise we are bound to the cron.js settings
-    sendHeartbeat();
-
-    callback(null);
+    syncConfigState(callback);
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clearTimeout(gAddMailDnsRecordsTimerId);
-    gAddMailDnsRecordsTimerId = null;
+    exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
 
     callback(null);
 }
 
+function isConfiguredSync() {
+    return gIsConfigured === true;
+}
+
+function isConfigured(callback) {
+    // set of rules to see if we have the configs required for cloudron to function
+    // note this checks for missing configs and not invalid configs
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(error);
+
+        if (!dnsConfig) return callback(null, false);
+
+        var isConfigured = (config.isCustomDomain() && dnsConfig.provider === 'route53') ||
+                        (!config.isCustomDomain() && dnsConfig.provider === 'caas');
+
+        callback(null, isConfigured);
+    });
+}
+
+function syncConfigState(callback) {
+    assert(!gIsConfigured);
+
+    callback = callback || NOOP_CALLBACK;
+
+    isConfigured(function (error, configured) {
+        if (error) return callback(error);
+
+        debug('syncConfigState: configured = %s', configured);
+
+        if (configured) {
+            exports.events.emit(exports.EVENT_CONFIGURED);
+        } else {
+            settings.events.once(settings.DNS_CONFIG_KEY, function () { syncConfigState(); }); // check again later
+        }
+
+        gIsConfigured = configured;
+
+        callback();
+    });
+}
+
 function setTimeZone(ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -139,7 +184,7 @@ function setTimeZone(ip, callback) {
         }
 
         if (!result.body.timezone) {
-            debug('No timezone in geoip response');
+            debug('No timezone in geoip response : %j', result.body);
             return callback(null);
         }
 
@@ -149,43 +194,38 @@ function setTimeZone(ip, callback) {
     });
 }
 
-function activate(username, password, email, name, ip, callback) {
+function activate(username, password, email, ip, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
     assert.strictEqual(typeof ip, 'string');
-    assert(!name || typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { });
+    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
-    if (!name) name = settings.getDefaultSync(settings.CLOUDRON_NAME_KEY);
-
-    settings.setCloudronName(name, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_NAME));
+    user.createOwner(username, password, email, function (error, userObject) {
+        if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
+        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
+        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
+        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        user.createOwner(username, password, email, function (error, userObject) {
-            if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-            if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-            if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-            if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-            clientdb.getByAppId('webadmin', function (error, result) {
+            // Also generate a token so the admin creation can also act as a login
+            var token = tokendb.generateToken();
+            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+
+            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-                // Also generate a token so the admin creation can also act as a login
-                var token = tokendb.generateToken();
-                var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+                // EE API is sync. do not keep the REST API reponse waiting
+                process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });
 
-                tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
-                    if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-                    callback(null, { token: token, expires: expires });
-                });
+                callback(null, { token: token, expires: expires });
             });
         });
     });
@@ -203,6 +243,8 @@ function getStatus(callback) {
             callback(null, {
                 activated: count !== 0,
                 version: config.version(),
+                boxVersionsUrl: config.get('boxVersionsUrl'),
+                apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool
                 cloudronName: cloudronName
             });
         });
@@ -230,10 +272,9 @@ function getCloudronDetails(callback) {
 function getConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    // TODO avoid pyramid of awesomeness with async
     getCloudronDetails(function (error, result) {
         if (error) {
-            console.error('Failed to fetch cloudron details.', error);
+            debug('Failed to fetch cloudron details.', error);
 
             // set fallback values to avoid dependency on appstore
             result = {
@@ -242,6 +283,10 @@ function getConfig(callback) {
             };
         }
 
+        // We rely at the moment on the size being specified in 512mb,1gb,...
+        // TODO provide that number from the appstore
+        var memory = bytes(result.size) || 0;
+
         settings.getCloudronName(function (error, cloudronName) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -261,6 +306,7 @@ function getConfig(callback) {
                     developerMode: developerMode,
                     region: result.region,
                     size: result.size,
+                    memory: memory,
                     cloudronName: cloudronName
                 });
             });
@@ -271,92 +317,117 @@ function getConfig(callback) {
 function sendHeartbeat() {
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
 
-    // TODO: this must be a POST
-    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
         if (error) debug('Error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
         else debug('Heartbeat sent to %s', url);
     });
 }
 
-function sendMailDnsRecordsRequest(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var DKIM_SELECTOR = 'mail';
-    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
-
+function readDkimPublicKeySync() {
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
 
-    if (publicKey === null) return callback(new Error('Error reading dkim public key'));
+    if (publicKey === null) {
+        debug('Error reading dkim public key.', safe.error);
+        return null;
+    }
 
     // remove header, footer and new lines
     publicKey = publicKey.split('\n').slice(1, -2).join('');
 
-    // note that dmarc requires special DNS records for external RUF and RUA
-    var records = [
-        // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
-        { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
-        // t=s limits the domainkey to this domain and not it's subdomains
-        { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', value: '"v=DKIM1; t=s; p=' + publicKey + '"' },
-        // DMARC requires special setup if report email id is in different domain
-        { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
-    ];
-
-    debug('sendMailDnsRecords request:%s', JSON.stringify(records));
-
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: records })
-        .end(function (error, res) {
-            if (error) return callback(error);
-
-            debug('sendMailDnsRecords status: %s', res.status);
-
-            if (res.status === 409) return callback(null); // already registered
-
-            if (res.status !== 201) return callback(new Error(util.format('Failed to add Mail DNS records: %s %j', res.status, res.body)));
-
-            return callback(null, res.body.ids);
-        });
+    return publicKey;
 }
 
-function addMailDnsRecords() {
-    if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
+function txtRecordsWithSpf(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    sendMailDnsRecordsRequest(function (error, ids) {
-        if (error) {
-            console.error('Mail DNS record addition failed', error);
-            gAddMailDnsRecordsTimerId = setTimeout(addMailDnsRecords, 30000);
-            return;
+    subdomains.get('', 'TXT', function (error, txtRecords) {
+        if (error) return callback(error);
+
+        debug('txtRecordsWithSpf: current txt records - %j', txtRecords);
+
+        var i, validSpf;
+
+        for (i = 0; i < txtRecords.length; i++) {
+            if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF
+
+            validSpf = txtRecords[i].indexOf(' a:' + config.fqdn() + ' ') !== -1;
+            break;
         }
 
-        debug('Added Mail DNS records successfully');
-        config.set('mailDnsRecordIds', ids);
+        if (validSpf) return callback(null, null);
+
+        if (i == txtRecords.length) {
+            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ~all"';
+        } else {
+            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
+        }
+
+        return callback(null, txtRecords);
     });
 }
 
-function setCertificate(certificate, key, callback) {
-    assert.strictEqual(typeof certificate, 'string');
-    assert.strictEqual(typeof key, 'string');
-    assert.strictEqual(typeof callback, 'function');
+function addDnsRecords() {
+    var callback = NOOP_CALLBACK;
 
-    debug('Updating certificates');
+    if (process.env.BOX_ENV === 'test') return callback();
 
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), certificate)) {
-        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+    if (gUpdatingDns) {
+        debug('addDnsRecords: dns update already in progress');
+        return callback();
+    }
+    gUpdatingDns = true;
+
+    var DKIM_SELECTOR = 'cloudron';
+    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
+
+    var dkimKey = readDkimPublicKeySync();
+    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
+
+    var nakedDomainRecord = { subdomain: '', type: 'A', values: [ sysinfo.getIp() ] };
+    var webadminRecord = { subdomain: 'my', type: 'A', values: [ sysinfo.getIp() ] };
+    // t=s limits the domainkey to this domain and not it's subdomains
+    var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
+    // DMARC requires special setup if report email id is in different domain
+    var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
+
+    var records = [ ];
+    if (config.isCustomDomain()) {
+        records.push(webadminRecord);
+        records.push(dkimRecord);
+    } else {
+        records.push(nakedDomainRecord);
+        records.push(webadminRecord);
+        records.push(dkimRecord);
+        records.push(dmarcRecord);
     }
 
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) {
-        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
-    }
+    debug('addDnsRecords: %j', records);
 
-    shell.sudo('setCertificate', [ RELOAD_NGINX_CMD ], function (error) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+    async.retry({ times: 10, interval: 20000 }, function (retryCallback) {
+        txtRecordsWithSpf(function (error, txtRecords) {
+            if (error) return retryCallback(error);
 
-        return callback(null);
+            if (txtRecords) records.push({ subdomain: '', type: 'TXT', values: txtRecords });
+
+            debug('addDnsRecords: will update %j', records);
+
+            async.mapSeries(records, function (record, iteratorCallback) {
+                subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
+            }, function (error, changeIds) {
+                if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
+                else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
+
+                retryCallback(error);
+            });
+        });
+    }, function (error) {
+        gUpdatingDns = false;
+
+        debug('addDnsRecords: done updating records with error:', error);
+
+        callback(error);
     });
 }
 
@@ -421,7 +492,7 @@ function update(boxUpdateInfo, callback) {
         debug('Starting upgrade');
         doUpgrade(boxUpdateInfo, function (error) {
             if (error) {
-                debug('Upgrade failed with error: %s', error);
+                console.error('Upgrade failed with error:', error);
                 locker.unlock(locker.OP_BOX_UPDATE);
             }
         });
@@ -429,7 +500,7 @@ function update(boxUpdateInfo, callback) {
         debug('Starting update');
         doUpdate(boxUpdateInfo, function (error) {
             if (error) {
-                debug('Update failed with error: %s', error);
+                console.error('Update failed with error:', error);
                 locker.unlock(locker.OP_BOX_UPDATE);
             }
         });
@@ -446,7 +517,7 @@ function doUpgrade(boxUpdateInfo, callback) {
         callback(e);
     }
 
-    progress.set(progress.UPDATE, 5, 'Create app and box backup for upgrade');
+    progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
 
     backupBoxAndApps(function (error) {
         if (error) return upgradeError(error);
@@ -456,7 +527,7 @@ function doUpgrade(boxUpdateInfo, callback) {
           .send({ version: boxUpdateInfo.version })
           .end(function (error, result) {
             if (error) return upgradeError(new Error('Error making upgrade request: ' + error));
-            if (result.status !== 202) return upgradeError(new Error('Server not ready to upgrade: ' + result.body));
+            if (result.status !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
 
             progress.set(progress.UPDATE, 10, 'Updating base system');
 
@@ -475,9 +546,9 @@ function doUpdate(boxUpdateInfo, callback) {
         callback(e);
     }
 
-    progress.set(progress.UPDATE, 5, 'Create box backup for update');
+    progress.set(progress.UPDATE, 5, 'Backing up for update');
 
-    backupBox(function (error) {
+    backupBoxAndApps(function (error) {
         if (error) return updateError(error);
 
         // fetch a signed sourceTarballUrl
@@ -486,25 +557,39 @@ function doUpdate(boxUpdateInfo, callback) {
           .end(function (error, result) {
             if (error) return updateError(new Error('Error fetching sourceTarballUrl: ' + error));
             if (result.status !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.status));
-            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + result.body));
+            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + JSON.stringify(result.body)));
 
             // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
             var args = {
                 sourceTarballUrl: result.body.url,
 
-                // IMPORTANT: if you change this, fix up argparser.sh as well. keep these sorted for readability
+                // this data is opaque to the installer
                 data: {
-                    apiServerOrigin: config.apiServerOrigin(),
-                    boxVersionsUrl: config.get('boxVersionsUrl'),
-                    fqdn: config.fqdn(),
-                    isCustomDomain: config.isCustomDomain(),
-                    restoreKey: null,
-                    restoreUrl: null,
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
                     token: config.token(),
+                    apiServerOrigin: config.apiServerOrigin(),
+                    webServerOrigin: config.webServerOrigin(),
+                    fqdn: config.fqdn(),
+                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    isCustomDomain: config.isCustomDomain(),
+
+                    appstore: {
+                        token: config.token(),
+                        apiServerOrigin: config.apiServerOrigin()
+                    },
+                    caas: {
+                        token: config.token(),
+                        apiServerOrigin: config.apiServerOrigin(),
+                        webServerOrigin: config.webServerOrigin()
+                    },
+                    tlsConfig: {
+                        provider: 'caas',
+                        cert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                        key: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    },
+
                     version: boxUpdateInfo.version,
-                    webServerOrigin: config.webServerOrigin()
+                    boxVersionsUrl: config.get('boxVersionsUrl')
                 }
             };
 
@@ -512,7 +597,7 @@ function doUpdate(boxUpdateInfo, callback) {
 
             superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
                 if (error) return updateError(error);
-                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + result.body));
+                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));
 
                 progress.set(progress.UPDATE, 10, 'Updating cloudron software');
 
@@ -530,6 +615,9 @@ function backup(callback) {
     var error = locker.lock(locker.OP_FULL_BACKUP);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    // clearing backup ensures tools can 'wait' on progress
+    progress.clear(progress.BACKUP);
+
     // start the backup operation in the background
     backupBoxAndApps(function (error) {
         if (error) console.error('backup failed.', error);
@@ -561,7 +649,7 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+    backups.getBackupUrl(null /* app */, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -569,14 +657,17 @@ function backupBoxWithAppBackupIds(appBackupIds, callback) {
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             debug('backup: successful');
 
-            callback(null, result.id);
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
+                callback(null, result.id);
+            });
         });
     });
 }
@@ -609,14 +700,14 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, 'Backing up app at ' + app.location);
-
-                if (error && error.reason === AppsError.BAD_STATE) {
-                    debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);
-                    backupId = app.lastBackupId;
+                if (error && error.reason !== AppsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
                 }
 
-                return iteratorCallback(null, backupId);
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
             });
         }, function appsBackedUp(error, backupIds) {
             if (error) {
@@ -624,7 +715,7 @@ function backupBoxAndApps(callback) {
                 return callback(error);
             }
 
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
 
             backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
                 progress.set(progress.BACKUP, 100, error ? error.message : '');

src/config.js

@@ -4,6 +4,8 @@
 
 exports = module.exports = {
     baseDir: baseDir,
+    dnsInSync: dnsInSync,
+    setDnsInSync: setDnsInSync,
 
     // values set here will be lost after a upgrade/update. use the sqlite database
     // for persistent values that need to be backed up
@@ -25,6 +27,7 @@ exports = module.exports = {
 
     // these values are derived
     adminOrigin: adminOrigin,
+    internalAdminOrigin: internalAdminOrigin,
     appFqdn: appFqdn,
     zoneName: zoneName,
 
@@ -52,6 +55,14 @@ function baseDir() {
 
 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
 
+function dnsInSync() {
+    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
+}
+
+function setDnsInSync(content) {
+    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
+}
+
 function saveSync() {
     fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
@@ -61,15 +72,15 @@ function initConfig() {
     data.fqdn = 'localhost';
 
     data.token = null;
-    data.mailServer = null;
     data.adminEmail = null;
-    data.mailDnsRecordIds = [ ];
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
     data.internalPort = 3001;
     data.ldapPort = 3002;
+    data.oauthProxyPort = 3003;
+    data.simpleAuthPort = 3004;
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -99,6 +110,9 @@ function initConfig() {
     saveSync();
 }
 
+// cleanup any old config file we have for tests
+if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
+
 initConfig();
 
 // set(obj) or set(key, value)
@@ -145,6 +159,10 @@ function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
 
+function internalAdminOrigin() {
+    return 'http://127.0.0.1:' + get('port');
+}
+
 function token() {
     return get('token');
 }
@@ -171,4 +189,3 @@ function database() {
 function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
-

src/constants.js

@@ -7,10 +7,6 @@ exports = module.exports = {
     ADMIN_NAME: 'Settings',
 
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
-    ADMIN_APPID: 'admin', // admin appid (settingsdb)
-
-    TEST_NAME: 'Test',
-    TEST_LOCATION: '',
-    TEST_CLIENT_ID: 'test'
+    ADMIN_APPID: 'admin' // admin appid (settingsdb)
 };
 

src/cron.js

@@ -8,8 +8,11 @@ exports = module.exports = {
 var apps = require('./apps.js'),
     assert = require('assert'),
     cloudron = require('./cloudron.js'),
+    config = require('./config.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
+    janitor = require('./janitor.js'),
+    scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
     updateChecker = require('./updatechecker.js');
 
@@ -17,9 +20,10 @@ var gAutoupdaterJob = null,
     gBoxUpdateCheckerJob = null,
     gAppUpdateCheckerJob = null,
     gHeartbeatJob = null,
-    gBackupJob = null;
-
-var gInitialized = false;
+    gBackupJob = null,
+    gCleanupTokensJob = null,
+    gDockerVolumeCleanerJob = null,
+    gSchedulerSyncJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 
@@ -34,27 +38,26 @@ var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gInitialized) return callback();
+    gHeartbeatJob = new CronJob({
+        cronTime: '00 */1 * * * *', // every minute
+        onTick: cloudron.sendHeartbeat,
+        start: true
+    });
+    cloudron.sendHeartbeat(); // latest unpublished version of CronJob has runOnInit
 
-    settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
-    settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
-
-    gInitialized = true;
-
-    recreateJobs(callback);
+    if (cloudron.isConfiguredSync()) {
+        recreateJobs(callback);
+    } else {
+        cloudron.events.on(cloudron.EVENT_ACTIVATED, recreateJobs);
+        callback();
+    }
 }
 
 function recreateJobs(unusedTimeZone, callback) {
     if (typeof unusedTimeZone === 'function') callback = unusedTimeZone;
 
     settings.getAll(function (error, allSettings) {
-        if (gHeartbeatJob) gHeartbeatJob.stop();
-        gHeartbeatJob = new CronJob({
-            cronTime: '00 */1 * * * *', // every minute
-            onTick: cloudron.sendHeartbeat,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
+        debug('Creating jobs with timezone %s', allSettings[settings.TIME_ZONE_KEY]);
 
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
@@ -80,14 +83,44 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCleanupTokensJob) gCleanupTokensJob.stop();
+        gCleanupTokensJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: janitor.cleanupTokens,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
+        gDockerVolumeCleanerJob = new CronJob({
+            cronTime: '00 00 */12 * * *', // every 12 hours
+            onTick: janitor.cleanupDockerVolumes,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
+        gSchedulerSyncJob = new CronJob({
+            cronTime: config.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
+            onTick: scheduler.sync,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
+        settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
         autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY]);
 
+        settings.events.removeListener(settings.TIME_ZONE_KEY, recreateJobs);
+        settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
+
         if (callback) callback();
     });
 }
 
 function autoupdatePatternChanged(pattern) {
     assert.strictEqual(typeof pattern, 'string');
+    assert(gBoxUpdateCheckerJob);
 
     debug('Auto update pattern changed to %s', pattern);
 
@@ -110,32 +143,41 @@ function autoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gBoxUpdateCheckerJob.cronTime.timeZone // hack
+        timeZone: gBoxUpdateCheckerJob.cronTime.zone // hack
     });
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (!gInitialized) return callback();
+    cloudron.events.removeListener(cloudron.EVENT_ACTIVATED, recreateJobs);
+
+    settings.events.removeListener(settings.TIME_ZONE_KEY, recreateJobs);
+    settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
 
     if (gAutoupdaterJob) gAutoupdaterJob.stop();
     gAutoupdaterJob = null;
 
-    gBoxUpdateCheckerJob.stop();
+    if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
     gBoxUpdateCheckerJob = null;
 
-    gAppUpdateCheckerJob.stop();
+    if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
     gAppUpdateCheckerJob = null;
 
-    gHeartbeatJob.stop();
+    if (gHeartbeatJob) gHeartbeatJob.stop();
     gHeartbeatJob = null;
 
-    gBackupJob.stop();
+    if (gBackupJob) gBackupJob.stop();
     gBackupJob = null;
 
-    gInitialized = false;
+    if (gCleanupTokensJob) gCleanupTokensJob.stop();
+    gCleanupTokensJob = null;
+
+    if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
+    gDockerVolumeCleanerJob = null;
+
+    if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
+    gSchedulerSyncJob = null;
 
     callback();
 }
-

src/developer.js

@@ -65,7 +65,7 @@ function issueDeveloperToken(user, callback) {
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'apps,settings,roleDeveloper', function (error) {
+    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
         callback(null, { token: token, expiresAt: expiresAt });

src/digitalocean.js

@@ -1,46 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    checkPtrRecord: checkPtrRecord
-};
-
-var assert = require('assert'),
-    debug = require('debug')('box:digitalocean'),
-    dns = require('native-dns');
-
-function checkPtrRecord(ip, fqdn, callback) {
-    assert(ip === null || typeof ip === 'string');
-    assert.strictEqual(typeof fqdn, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('checkPtrRecord: ' + ip);
-
-    if (!ip) return callback(new Error('Network down'));
-
-    dns.resolve4('ns1.digitalocean.com', function (error, rdnsIps) {
-        if (error || rdnsIps.length === 0) return callback(new Error('Failed to query DO DNS'));
-
-        var reversedIp = ip.split('.').reverse().join('.');
-
-        var req = dns.Request({
-            question: dns.Question({ name: reversedIp + '.in-addr.arpa', type: 'PTR' }),
-            server: { address: rdnsIps[0] },
-            timeout: 5000
-        });
-
-        req.on('timeout', function () { return callback(new Error('Timedout')); });
-
-        req.on('message', function (error, message) {
-            if (error || !message.answer || message.answer.length === 0) return callback(new Error('Failed to query PTR'));
-
-            debug('checkPtrRecord: Actual:%s Expecting:%s', message.answer[0].data, fqdn);
-            callback(null, message.answer[0].data === fqdn);
-        });
-
-        req.send();
-    });
-}
-
-

src/dns/caas.js

@@ -0,0 +1,136 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    update: update,
+    getChangeStatus: getChangeStatus,
+    get: get
+};
+
+var assert = require('assert'),
+    config = require('../config.js'),
+    debug = require('debug')('box:dns/caas'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    superagent = require('superagent'),
+    util = require('util'),
+    _ = require('underscore');
+
+function add(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    var fqdn = subdomain !== '' && type === 'TXT' ? subdomain + '.' + config.fqdn() : config.appFqdn(subdomain);
+
+    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .post(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.status !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.changeId);
+        });
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var fqdn = subdomain !== '' && type === 'TXT' ? subdomain + '.' + config.fqdn() : config.appFqdn(subdomain);
+
+    debug('get: zoneName: %s subdomain: %s type: %s fqdn: %s', zoneName, subdomain, type, fqdn);
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
+        .query({ token: dnsConfig.token, type: type })
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.values);
+        });
+}
+
+function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        if (_.isEqual(values, result)) return callback();
+
+        add(dnsConfig, zoneName, subdomain, type, values, callback);
+    });
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .del(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.status === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND));
+            if (result.status !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null);
+        });
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
+        .query({ token: dnsConfig.token })
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.status);
+        });
+
+}

src/dns/route53.js

@@ -0,0 +1,214 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    get: get,
+    del: del,
+    update: update,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('../config.js'),
+    debug = require('debug')('box:dns/route53'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    util = require('util'),
+    _ = require('underscore');
+
+function getDnsCredentials(dnsConfig) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    var credentials = {
+        accessKeyId: dnsConfig.accessKeyId,
+        secretAccessKey: dnsConfig.secretAccessKey,
+        region: dnsConfig.region
+    };
+
+    if (dnsConfig.endpoint) credentials.endpoint = new AWS.Endpoint(dnsConfig.endpoint);
+
+    return credentials;
+}
+
+function getZoneByName(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    route53.listHostedZones({}, function (error, result) {
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+
+        var zone = result.HostedZones.filter(function (zone) {
+            return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+        })[0];
+
+        if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
+
+        callback(null, zone);
+    });
+}
+
+function add(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var records = values.map(function (v) { return { Value: v }; });
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'UPSERT',
+                    ResourceRecordSet: {
+                        Type: type,
+                        Name: fqdn,
+                        ResourceRecords: records,
+                        TTL: 1
+                    }
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'PriorRequestNotComplete') {
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            } else if (error) {
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+            }
+
+            callback(null, result.ChangeInfo.Id);
+        });
+    });
+}
+
+function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        if (_.isEqual(values, result)) return callback();
+
+        add(dnsConfig, zoneName, subdomain, type, values, callback);
+    });
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var params = {
+            HostedZoneId: zone.Id,
+            MaxItems: '1',
+            StartRecordName: (subdomain ? subdomain + '.' : '') + zoneName + '.',
+            StartRecordType: type
+        };
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.listResourceRecordSets(params, function (error, result) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
+            if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
+
+            var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
+
+            callback(null, values);
+        });
+    });
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var records = values.map(function (v) { return { Value: v }; });
+
+        var resourceRecordSet = {
+            Name: fqdn,
+            Type: type,
+            ResourceRecords: records,
+            TTL: 1
+        };
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'DELETE',
+                    ResourceRecordSet: resourceRecordSet
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                debug('delSubdomain: resource record set not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+            } else if (error && error.code === 'NoSuchHostedZone') {
+                debug('delSubdomain: hosted zone not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+            } else if (error && error.code === 'PriorRequestNotComplete') {
+                debug('delSubdomain: resource is still busy', error);
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+            } else if (error && error.code === 'InvalidChangeBatch') {
+                debug('delSubdomain: invalid change batch. No such record to be deleted.');
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+            } else if (error) {
+                debug('delSubdomain: error', error);
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            }
+
+            callback(null);
+        });
+    });
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    route53.getChange({ Id: changeId }, function (error, result) {
+        if (error) return callback(error);
+
+        callback(null, result.ChangeInfo.Status);
+    });
+}
+

src/docker.js

@@ -1,42 +1,347 @@
 'use strict';
 
-var Docker = require('dockerode'),
-    fs = require('fs'),
-    os = require('os'),
-    path = require('path'),
-    url = require('url');
+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:src/docker.js'),
+    Docker = require('dockerode'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    util = require('util'),
+    _ = require('underscore');
 
-exports = module.exports = (function () {
+exports = module.exports = {
+    connection: connectionInstance(),
+    downloadImage: downloadImage,
+    createContainer: createContainer,
+    startContainer: startContainer,
+    stopContainer: stopContainer,
+    stopContainers: stopContainers,
+    deleteContainer: deleteContainer,
+    deleteImage: deleteImage,
+    deleteContainers: deleteContainers,
+    createSubcontainer: createSubcontainer
+};
+
+function connectionInstance() {
     var docker;
-    var options = connectOptions(); // the real docker
 
     if (process.env.BOX_ENV === 'test') {
         // test code runs a docker proxy on this port
         docker = new Docker({ host: 'http://localhost', port: 5687 });
+
+        // proxy code uses this to route to the real docker
+        docker.options = { socketPath: '/var/run/docker.sock' };
     } else {
-        docker = new Docker(options);
+        docker = new Docker({ socketPath: '/var/run/docker.sock' });
     }
 
-    // proxy code uses this to route to the real docker
-    docker.options = options;
-
     return docker;
-})();
-
-function connectOptions() {
-    if (os.platform() === 'linux') return { socketPath: '/var/run/docker.sock' };
-
-    // boot2docker configuration
-    var DOCKER_CERT_PATH = process.env.DOCKER_CERT_PATH || path.join(process.env.HOME, '.boot2docker/certs/boot2docker-vm');
-    var DOCKER_HOST = process.env.DOCKER_HOST || 'tcp://192.168.59.103:2376';
-
-    return {
-        protocol: 'https',
-        host: url.parse(DOCKER_HOST).hostname,
-        port: url.parse(DOCKER_HOST).port,
-        ca: fs.readFileSync(path.join(DOCKER_CERT_PATH, 'ca.pem')),
-        cert: fs.readFileSync(path.join(DOCKER_CERT_PATH, 'cert.pem')),
-        key: fs.readFileSync(path.join(DOCKER_CERT_PATH, 'key.pem'))
-    };
 }
 
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? (app.location || '(bare)') : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function targetBoxVersion(manifest) {
+    if ('targetBoxVersion' in manifest) return manifest.targetBoxVersion;
+
+    if ('minBoxVersion' in manifest) return manifest.minBoxVersion;
+
+    return '0.0.1';
+}
+
+function pullImage(manifest, callback) {
+    var docker = exports.connection;
+
+    docker.pull(manifest.dockerImage, function (err, stream) {
+        if (err) return callback(new Error('Error connecting to docker. statusCode: %s' + err.statusCode));
+
+        // https://github.com/dotcloud/docker/issues/1074 says each status message
+        // is emitted as a chunk
+        stream.on('data', function (chunk) {
+            var data = safe.JSON.parse(chunk) || { };
+            debug('pullImage %s: %j', manifest.id, data);
+
+            // The information here is useless because this is per layer as opposed to per image
+            if (data.status) {
+            } else if (data.error) {
+                debug('pullImage error %s: %s', manifest.id, data.errorDetail.message);
+            }
+        });
+
+        stream.on('end', function () {
+            debug('downloaded image %s of %s successfully', manifest.dockerImage, manifest.id);
+
+            var image = docker.getImage(manifest.dockerImage);
+
+            image.inspect(function (err, data) {
+                if (err) return callback(new Error('Error inspecting image:' + err.message));
+                if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
+                if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
+
+                debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
+
+                callback(null);
+            });
+        });
+
+        stream.on('error', function (error) {
+            debug('error pulling image %s of %s: %j', manifest.dockerImage, manifest.id, error);
+
+            callback(error);
+        });
+    });
+}
+
+function downloadImage(manifest, callback) {
+    assert.strictEqual(typeof manifest, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('downloadImage %s %s', manifest.id, manifest.dockerImage);
+
+    var attempt = 1;
+
+    async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
+        debug('Downloading image %s %s. attempt: %s', manifest.id, manifest.dockerImage, attempt++);
+
+        pullImage(manifest, function (error) {
+            if (error) console.error(error);
+
+            retryCallback(error);
+        });
+    }, callback);
+}
+
+function createSubcontainer(app, name, cmd, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof name, 'string');
+    assert(!cmd || util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection,
+        isAppContainer = !cmd;
+
+    var manifest = app.manifest;
+    var exposedPorts = {}, dockerPortBindings = { };
+    var stdEnv = [
+        'CLOUDRON=1',
+        'WEBADMIN_ORIGIN=' + config.adminOrigin(),
+        'API_ORIGIN=' + config.adminOrigin(),
+        'APP_ORIGIN=https://' + config.appFqdn(app.location),
+        'APP_DOMAIN=' + config.appFqdn(app.location)
+    ];
+
+    // docker portBindings requires ports to be exposed
+    exposedPorts[manifest.httpPort + '/tcp'] = {};
+
+    dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: '127.0.0.1', HostPort: app.httpPort + '' } ];
+
+    var portEnv = [];
+    for (var e in app.portBindings) {
+        var hostPort = app.portBindings[e];
+        var containerPort = manifest.tcpPorts[e].containerPort || hostPort;
+
+        exposedPorts[containerPort + '/tcp'] = {};
+        portEnv.push(e + '=' + hostPort);
+
+        dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
+    }
+
+    var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+
+    addons.getEnvironment(app, function (error, addonEnv) {
+        if (error) return callback(new Error('Error getting addon environment : ' + error));
+
+        var containerOptions = {
+            name: name, // used for filtering logs
+            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
+            Hostname: semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location),
+            Tty: isAppContainer,
+            Image: app.manifest.dockerImage,
+            Cmd: cmd,
+            Env: stdEnv.concat(addonEnv).concat(portEnv),
+            ExposedPorts: isAppContainer ? exposedPorts : { },
+            Volumes: { // see also ReadonlyRootfs
+                '/tmp': {},
+                '/run': {}
+            },
+            Labels: {
+                "location": app.location,
+                "appId": app.id,
+                "isSubcontainer": String(!isAppContainer)
+            },
+            HostConfig: {
+                Binds: addons.getBindsSync(app, app.manifest.addons),
+                Memory: memoryLimit / 2,
+                MemorySwap: memoryLimit, // Memory + Swap
+                PortBindings: isAppContainer ? dockerPortBindings : { },
+                PublishAllPorts: false,
+                ReadonlyRootfs: semver.gte(targetBoxVersion(app.manifest), '0.0.66'), // see also Volumes in startContainer
+                Links: addons.getLinksSync(app, app.manifest.addons),
+                RestartPolicy: {
+                    "Name": isAppContainer ? "always" : "no",
+                    "MaximumRetryCount": 0
+                },
+                CpuShares: 512, // relative to 1024 for system processes
+                VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
+                SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
+            }
+        };
+        containerOptions = _.extend(containerOptions, options);
+
+        // older versions wanted a writable /var/log
+        if (semver.lte(targetBoxVersion(app.manifest), '0.0.71')) containerOptions.Volumes['/var/log'] = {};
+
+        debugApp(app, 'Creating container for %s with options %j', app.manifest.dockerImage, containerOptions);
+
+        docker.createContainer(containerOptions, callback);
+    });
+}
+
+function createContainer(app, callback) {
+    createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback);
+}
+
+function startContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    var container = docker.getContainer(containerId);
+    debug('Starting container %s', containerId);
+
+    container.start(function (error) {
+        if (error && error.statusCode !== 304) return callback(new Error('Error starting container :' + error));
+
+        return callback(null);
+    });
+}
+
+function stopContainer(containerId, callback) {
+    assert(!containerId || typeof containerId === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!containerId) {
+        debug('No previous container to stop');
+        return callback();
+    }
+
+    var docker = exports.connection;
+    var container = docker.getContainer(containerId);
+    debug('Stopping container %s', containerId);
+
+    var options = {
+        t: 10 // wait for 10 seconds before killing it
+    };
+
+    container.stop(options, function (error) {
+        if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error stopping container:' + error));
+
+        debug('Waiting for container ' + containerId);
+
+        container.wait(function (error, data) {
+            if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error waiting on container:' + error));
+
+            debug('Container %s stopped with status code [%s]', containerId, data ? String(data.StatusCode) : '');
+
+            return callback(null);
+        });
+    });
+}
+
+function deleteContainer(containerId, callback) {
+    assert(!containerId || typeof containerId === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('deleting container %s', containerId);
+
+    if (containerId === null) return callback(null);
+
+    var docker = exports.connection;
+    var container = docker.getContainer(containerId);
+
+    var removeOptions = {
+        force: true, // kill container if it's running
+        v: true // removes volumes associated with the container (but not host mounts)
+    };
+
+    container.remove(removeOptions, function (error) {
+        if (error && error.statusCode === 404) return callback(null);
+
+        if (error) debug('Error removing container %s : %j', containerId, error);
+
+        callback(error);
+    });
+}
+
+function deleteContainers(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    debug('deleting containers of %s', appId);
+
+    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            deleteContainer(container.Id, iteratorDone);
+        }, callback);
+    });
+}
+
+function stopContainers(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    debug('stopping containers of %s', appId);
+
+    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            stopContainer(container.Id, iteratorDone);
+        }, callback);
+    });
+}
+
+function deleteImage(manifest, callback) {
+    assert(!manifest || typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var dockerImage = manifest ? manifest.dockerImage : null;
+    if (!dockerImage) return callback(null);
+
+    var docker = exports.connection;
+
+    docker.getImage(dockerImage).inspect(function (error, result) {
+        if (error && error.statusCode === 404) return callback(null);
+
+        if (error) return callback(error);
+
+        var removeOptions = {
+            force: true,
+            noprune: false
+        };
+
+         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
+        docker.getImage(result.Id).remove(removeOptions, function (error) {
+            if (error && error.statusCode === 404) return callback(null);
+            if (error && error.statusCode === 409) return callback(null); // another container using the image
+
+            if (error) debug('Error removing image %s : %j', dockerImage, error);
+
+            callback(error);
+        });
+    });
+}

src/janitor.js

@@ -0,0 +1,103 @@
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    authcodedb = require('./authcodedb.js'),
+    debug = require('debug')('box:src/janitor'),
+    docker = require('./docker.js').connection,
+    tokendb = require('./tokendb.js');
+
+exports = module.exports = {
+    cleanupTokens: cleanupTokens,
+    cleanupDockerVolumes: cleanupDockerVolumes
+};
+
+var NOOP_CALLBACK = function () { };
+
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+
+            callback();
+        });
+    };
+}
+
+function cleanupExpiredTokens(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired tokens.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupExpiredAuthCodes(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    authcodedb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired authcodes.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupTokens(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    debug('Cleaning up expired tokens');
+
+    async.series([
+        ignoreError(cleanupExpiredTokens),
+        ignoreError(cleanupExpiredAuthCodes)
+    ], callback);
+}
+
+function cleanupTmpVolume(containerInfo, callback) {
+    assert.strictEqual(typeof containerInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var cmd = 'find /tmp -mtime +10 -exec rm -rf {} +'.split(' '); // 10 days old
+
+    debug('cleanupTmpVolume %j', containerInfo.Names);
+
+    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+
+        execContainer.start(function(err, stream) {
+            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+
+            stream.on('error', callback);
+            stream.on('end', callback);
+
+            stream.setEncoding('utf8');
+            stream.pipe(process.stdout);
+        });
+    });
+}
+
+function cleanupDockerVolumes(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Cleaning up docker volumes');
+
+    docker.listContainers({ all: 0 }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            cleanupTmpVolume(container, function (error) {
+                if (error) debug('Error cleaning tmp: %s', error);
+
+                iteratorDone(); // intentionally ignore error
+            });
+        }, callback);
+    });
+}

src/ldap.js

@@ -1,7 +1,8 @@
 'use strict';
 
 exports = module.exports = {
-    start: start
+    start: start,
+    stop: stop
 };
 
 var assert = require('assert'),
@@ -25,10 +26,10 @@ var gLogger = {
 };
 
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
-var GROUP_ADMINS_DN = 'cn=admin,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
 function start(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     gServer = ldap.createServer({ log: gLogger });
 
@@ -62,7 +63,6 @@ function start(callback) {
 
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
                     res.send(tmp);
-                    debug('ldap user send:', tmp);
                 }
             });
 
@@ -99,7 +99,6 @@ function start(callback) {
 
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
                     res.send(tmp);
-                    debug('ldap group send:', tmp);
                 }
             });
 
@@ -107,8 +106,14 @@ function start(callback) {
         });
     });
 
-    gServer.bind('dc=cloudron', function(req, res, next) {
-        debug('ldap bind: %s', req.dn.toString());
+    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
+        // TODO: validate password
+        debug('ldap application bind: %s', req.dn.toString());
+        res.end();
+    });
+
+    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
+        debug('ldap user bind: %s', req.dn.toString());
 
         if (!req.dn.rdns[0].cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
@@ -123,3 +128,11 @@ function start(callback) {
 
     gServer.listen(config.get('ldapPort'), callback);
 }
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gServer.close();
+
+    callback();
+}

src/locker.js

@@ -9,6 +9,7 @@ function Locker() {
     this._operation = null;
     this._timestamp = null;
     this._watcherId = -1;
+    this._lockDepth = 0; // recursive locks
 }
 util.inherits(Locker, EventEmitter);
 
@@ -24,6 +25,7 @@ Locker.prototype.lock = function (operation) {
     if (this._operation !== null) return new Error('Already locked for ' + this._operation);
 
     this._operation = operation;
+    ++this._lockDepth;
     this._timestamp = new Date();
     var that = this;
     this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
@@ -35,17 +37,31 @@ Locker.prototype.lock = function (operation) {
     return null;
 };
 
+Locker.prototype.recursiveLock = function (operation) {
+    if (this._operation === operation) {
+        ++this._lockDepth;
+        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
+        return null;
+    }
+
+    return this.lock(operation);
+};
+
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
-    debug('Released : %s', this._operation);
+    if (--this._lockDepth === 0) {
+        debug('Released : %s', this._operation);
 
-    this._operation = null;
-    this._timestamp = null;
-    clearInterval(this._watcherId);
-    this._watcherId = -1;
+        this._operation = null;
+        this._timestamp = null;
+        clearInterval(this._watcherId);
+        this._watcherId = -1;
+    } else {
+        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
+    }
 
     this.emit('unlocked', operation);
 

src/mail_templates/crash_notification.ejs

@@ -2,7 +2,7 @@
 
 Dear Cloudron Team,
 
-unfortunately the <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
 
 Please see some excerpt of the logs below.
 
@@ -11,7 +11,7 @@ Your Cloudron
 
 -------------------------------------
 
-<%= context %>
+<%- context %>
 
 <% } else { %>
 

src/mail_templates/welcome_user.ejs

@@ -2,9 +2,9 @@
 
 Dear <%= user.username %>,
 
-I am excited to welcome you to my Cloudron <%= fqdn %>!
+Welcome to my Cloudron <%= fqdn %>!
 
-The Cloudron is our own Private Cloud. You can read more about it
+The Cloudron is our own Smart Server. You can read more about it
 at https://www.cloudron.io.
 
            You username is '<%= user.username %>'

src/mailer.js

@@ -25,16 +25,16 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
+    cloudron = require('./cloudron.js'),
     config = require('./config.js'),
     debug = require('debug')('box:mailer'),
-    digitalocean = require('./digitalocean.js'),
-    docker = require('./docker.js'),
+    dns = require('native-dns'),
+    docker = require('./docker.js').connection,
     ejs = require('ejs'),
     nodemailer = require('nodemailer'),
     path = require('path'),
     safe = require('safetydance'),
     smtpTransport = require('nodemailer-smtp-transport'),
-    sysinfo = require('./sysinfo.js'),
     userdb = require('./userdb.js'),
     util = require('util'),
     _ = require('underscore');
@@ -48,13 +48,20 @@ var gMailQueue = [ ],
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    checkDns();
+    if (cloudron.isConfiguredSync()) {
+        checkDns();
+    } else {
+        cloudron.events.on(cloudron.EVENT_CONFIGURED, checkDns);
+    }
+
     callback(null);
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    cloudron.events.removeListener(cloudron.EVENT_CONFIGURED, checkDns);
+
     // TODO: interrupt processQueue as well
     clearTimeout(gCheckDnsTimerId);
     gCheckDnsTimerId = null;
@@ -65,20 +72,76 @@ function uninitialize(callback) {
     callback(null);
 }
 
+function getTxtRecords(callback) {
+    dns.resolveNs(config.zoneName(), function (error, nameservers) {
+        if (error || !nameservers) return callback(error || new Error('Unable to get nameservers'));
+
+        var nameserver = nameservers[0];
+
+        dns.resolve4(nameserver, function (error, nsIps) {
+            if (error || !nsIps || nsIps.length === 0) return callback(error);
+
+            var req = dns.Request({
+                question: dns.Question({ name: config.fqdn(), type: 'TXT' }),
+                server: { address: nsIps[0] },
+                timeout: 5000
+            });
+
+            req.on('timeout', function () { return callback(new Error('ETIMEOUT')); });
+
+            req.on('message', function (error, message) {
+                if (error || !message.answer || message.answer.length === 0) return callback(null, null);
+
+                var records = message.answer.map(function (a) { return a.data[0]; });
+                callback(null, records);
+            });
+
+            req.send();
+        });
+    });
+}
+
 function checkDns() {
-    digitalocean.checkPtrRecord(sysinfo.getIp(), config.fqdn(), function (error, ok) {
-        if (error || !ok) {
-            debug('PTR record not setup yet');
-            gCheckDnsTimerId = setTimeout(checkDns, 10000);
+    getTxtRecords(function (error, records) {
+        if (error || !records) {
+            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.fqdn(), error, records);
+            gCheckDnsTimerId = setTimeout(checkDns, 60000);
             return;
         }
 
+        var allowedToSendMail = false;
+
+        for (var i = 0; i < records.length; i++) {
+            if (records[i].indexOf('v=spf1 ') !== 0) continue; // not SPF
+
+            allowedToSendMail = records[i].indexOf('a:' + config.fqdn()) !== -1;
+            break; // only one SPF record can exist (https://support.google.com/a/answer/4568483?hl=en)
+        }
+
+        if (!allowedToSendMail) {
+            debug('checkDns: SPF records disallow sending email from cloudron. %j', records);
+            gCheckDnsTimerId = setTimeout(checkDns, 60000);
+            return;
+        }
+
+        debug('checkDns: SPF check passed. commencing mail processing');
         gDnsReady = true;
         processQueue();
     });
 }
 
 function processQueue() {
+    assert(gDnsReady);
+
+    sendMails(gMailQueue);
+    gMailQueue = [ ];
+}
+
+// note : this function should NOT access the database. it is called by the crashnotifier
+// which does not initialize mailer or the databse
+function sendMails(queue) {
+    assert(util.isArray(queue));
+
     docker.getContainer('mail').inspect(function (error, data) {
         if (error) return console.error(error);
 
@@ -87,15 +150,12 @@ function processQueue() {
 
         var transport = nodemailer.createTransport(smtpTransport({
             host: mailServerIp,
-            port: 25
+            port: 2500 // this value comes from mail container
         }));
 
-        var mailQueueCopy = gMailQueue;
-        gMailQueue = [ ];
+        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
 
-        debug('Processing mail queue of size %d', mailQueueCopy.length);
-
-        async.mapSeries(mailQueueCopy, function iterator(mailOptions, callback) {
+        async.mapSeries(queue, function iterator(mailOptions, callback) {
             transport.sendMail(mailOptions, function (error) {
                 if (error) return console.error(error); // TODO: requeue?
                 debug('Email sent to ' + mailOptions.to);
@@ -269,6 +329,8 @@ function appUpdateAvailable(app, updateInfo) {
     });
 }
 
+// this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
+// crashnotifier should be able to send mail when there is no db
 function sendCrashNotification(program, context) {
     assert.strictEqual(typeof program, 'string');
     assert.strictEqual(typeof context, 'string');
@@ -280,7 +342,7 @@ function sendCrashNotification(program, context) {
         text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
     };
 
-    enqueue(mailOptions);
+    sendMails([ mailOptions ]);
 }
 
 function sendFeedback(user, type, subject, description) {

src/middleware/contentType.js

@@ -1,8 +0,0 @@
-'use strict';
-
-module.exports = function contentType(type) {
-    return function (req, res, next) {
-        res.setHeader('Content-Type', type);
-        next();
-    };
-};

src/middleware/index.js

@@ -1,7 +1,6 @@
 'use strict';
 
 exports = module.exports = {
-    contentType: require('./contentType'),
     cookieParser: require('cookie-parser'),
     cors: require('./cors'),
     csrf: require('csurf'),

src/oauth2views/callback.ejs

@@ -1,4 +1,4 @@
-<% include header %>
+<!-- callback tester -->
 
 <script>
 
@@ -35,13 +35,11 @@ args.forEach(function (arg) {
 });
 
 if (code && redirectURI) {
-    window.location.href = redirectURI + '?code=' + code + (state ? '&state=' + state : '');
+    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'code=' + code + (state ? '&state=' + state : '');
 } else if (redirectURI && accessToken) {
-    window.location.href = redirectURI + '?token=' + accessToken + (state ? '&state=' + state : '');
+    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'token=' + accessToken + (state ? '&state=' + state : '');
 } else {
     window.location.href = '/api/v1/session/login';
 }
 
 </script>
-
-<% include footer %>;

src/oauth2views/dialog.ejs

@@ -1,38 +0,0 @@
-<% include header %>
-
-<form action="/api/v1/oauth/dialog/authorize/decision" method="post">
-  <input name="transaction_id" type="hidden" value="<%= transactionID %>">
-  <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-
-  <div class="row">
-    <div class="col-md-3"></div>
-    <div class="col-md-6">
-      <div class="container-fluid">
-        <div class="row">
-          <div class="col-sm-12">
-            Hi <%= user.username %>!
-          </div>
-        </div>
-        <div class="row">
-          <div class="col-sm-12">
-            <b><%= client.name %></b> is requesting access to your account.
-          </div>
-        </div>
-        <div class="row">
-          <div class="col-sm-12">
-            Do you approve?
-          </div>
-        </div>
-        <div class="row">
-          <div class="col-sm-12">
-            <input class="btn btn-danger btn-outline" type="submit" value="Deny" name="cancel" id="deny"/>
-            <input class="btn btn-success btn-outline" type="submit" value="Allow"/>
-          </div>
-        </div>
-      </div>
-    </div>
-    <div class="col-md-3"></div>
-  </div>
-</form>
-
-<% include footer %>

src/oauth2views/error.ejs

@@ -1,5 +1,7 @@
 <% include header %>
 
+<!-- error tester -->
+
 <br/>
 
 <div class="container">

src/oauth2views/header.ejs

@@ -6,6 +6,8 @@
 
     <title> Cloudron Login </title>
 
+    <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="https://fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -26,3 +28,13 @@
 </head>
 
 <body class="oauth">
+
+    <!-- Navigation -->
+    <nav class="navbar navbar-default navbar-static-top shadow" role="navigation" style="margin-bottom: 0">
+        <div class="container-fluid">
+            <div class="navbar-header">
+                <span class="navbar-brand navbar-brand-icon"><img src="/api/v1/cloudron/avatar" width="40" height="40"/></span>
+                <span class="navbar-brand">Cloudron</span>
+            </div>
+        </div>
+    </nav>

src/oauth2views/login.ejs

@@ -1,5 +1,7 @@
 <% include header %>
 
+<!-- login tester -->
+
 <div class="container">
     <div class="row">
         <div class="col-md-6 col-md-offset-3">
@@ -7,7 +9,7 @@
                 <div class="row">
                     <div class="col-md-12" style="text-align: center;">
                         <img width="128" height="128" src="<%= applicationLogo %>"/>
-                        <h1>Login to <%= applicationName %> on <%= cloudronName %></h1>
+                        <h1><small>Login to</small> <%= applicationName %></h1>
                         <br/>
                     </div>
                 </div>

src/oauthproxy.js

@@ -0,0 +1,195 @@
+'use strict';
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var appdb = require('./appdb.js'),
+    assert = require('assert'),
+    clientdb = require('./clientdb.js'),
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:proxy'),
+    express = require('express'),
+    http = require('http'),
+    proxy = require('proxy-middleware'),
+    session = require('cookie-session'),
+    superagent = require('superagent'),
+    tokendb = require('./tokendb.js'),
+    url = require('url'),
+    uuid = require('node-uuid');
+
+var gSessions = {};
+var gProxyMiddlewareCache = {};
+var gHttpServer = null;
+
+var CALLBACK_URI = '/callback';
+
+function clearSession(req) {
+    delete gSessions[req.session.id];
+
+    req.session.id = uuid.v4();
+    gSessions[req.session.id] = {};
+
+    req.sessionData = gSessions[req.session.id];
+
+}
+
+function attachSessionData(req, res, next) {
+    assert.strictEqual(typeof req.session, 'object');
+
+    if (!req.session.id || !gSessions[req.session.id]) clearSession(req);
+
+    // attach the session data to the requeset
+    req.sessionData = gSessions[req.session.id];
+
+    next();
+}
+
+function verifySession(req, res, next) {
+    assert.strictEqual(typeof req.sessionData, 'object');
+
+    if (!req.sessionData.accessToken) {
+        req.authenticated = false;
+        return next();
+    }
+
+    tokendb.get(req.sessionData.accessToken, function (error, token) {
+        if (error)  {
+            if (error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+            clearSession(req);
+            req.authenticated = false;
+        } else {
+            req.authenticated = true;
+        }
+
+        next();
+    });
+}
+
+function authenticate(req, res, next) {
+    // proceed if we are authenticated
+    if (req.authenticated) return next();
+
+    if (req.path === CALLBACK_URI && req.sessionData.returnTo) {
+        // exchange auth code for an access token
+        var query = {
+            response_type: 'token',
+            client_id: req.sessionData.clientId
+        };
+
+        var data = {
+            grant_type: 'authorization_code',
+            code: req.query.code,
+            redirect_uri: req.sessionData.returnTo,
+            client_id: req.sessionData.clientId,
+            client_secret: req.sessionData.clientSecret
+        };
+
+        // use http admin origin so that it works with self-signed certs
+        superagent
+            .post(config.internalAdminOrigin() + '/api/v1/oauth/token')
+            .query(query).send(data)
+            .end(function (error, result) {
+            if (error) {
+                console.error(error);
+                return res.send(500, 'Unable to contact the oauth server.');
+            }
+            if (result.statusCode !== 200) {
+                console.error('Failed to exchange auth code for a token.', result.statusCode, result.body);
+                return res.send(500, 'Failed to exchange auth code for a token.');
+            }
+
+            req.sessionData.accessToken = result.body.access_token;
+
+            debug('user verified.');
+
+            // now redirect to the actual initially requested URL
+            res.redirect(req.sessionData.returnTo);
+        });
+    } else {
+        var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
+
+        if (!Number.isFinite(port)) {
+            console.error('Failed to parse nginx proxy header to get app port.');
+            return res.send(500, 'Routing error. No forwarded port.');
+        }
+
+        debug('begin verifying user for app on port %s.', port);
+
+        appdb.getByHttpPort(port, function (error, result) {
+            if (error) {
+                console.error('Unknown app.', error);
+                return res.send(500, 'Unknown app.');
+            }
+
+            clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
+                if (error) {
+                    console.error('Unkonwn OAuth client.', error);
+                    return res.send(500, 'Unknown OAuth client.');
+                }
+
+                req.sessionData.port = port;
+                req.sessionData.returnTo = result.redirectURI + req.path;
+                req.sessionData.clientId = result.id;
+                req.sessionData.clientSecret = result.clientSecret;
+
+                var callbackUrl = result.redirectURI + CALLBACK_URI;
+                var scope = 'profile';
+                var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + result.id + '&redirect_uri=' + callbackUrl + '&scope=' + scope;
+
+                debug('begin OAuth flow for client %s.', result.name);
+
+                // begin the OAuth flow
+                res.redirect(oauthLogin);
+            });
+        });
+    }
+}
+
+function forwardRequestToApp(req, res, next) {
+    var port = req.sessionData.port;
+
+    debug('proxy request for port %s with path %s.', port, req.path);
+
+    var proxyMiddleware = gProxyMiddlewareCache[port];
+    if (!proxyMiddleware) {
+        console.log('Adding proxy middleware for port %d', port);
+
+        proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
+        gProxyMiddlewareCache[port] = proxyMiddleware;
+    }
+
+    proxyMiddleware(req, res, next);
+}
+
+function initializeServer() {
+    var app = express();
+    var httpServer = http.createServer(app);
+
+    httpServer.on('error', console.error);
+
+    app
+        .use(session({ keys: ['blue', 'cheese', 'is', 'something'] }))
+        .use(attachSessionData)
+        .use(verifySession)
+        .use(authenticate)
+        .use(forwardRequestToApp);
+
+    return httpServer;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gHttpServer = initializeServer();
+
+    gHttpServer.listen(config.get('oauthProxyPort'), callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gHttpServer.close(callback);
+}

src/paths.js

@@ -12,6 +12,9 @@ exports = module.exports = {
     NGINX_CERT_DIR: path.join(config.baseDir(), 'data/nginx/cert'),
 
     ADDON_CONFIG_DIR: path.join(config.baseDir(), 'data/addons'),
+    SCHEDULER_FILE: path.join(config.baseDir(), 'data/addons/scheduler.json'),
+
+    DNS_IN_SYNC_FILE: path.join(config.baseDir(), 'data/dns_in_sync'),
 
     COLLECTD_APPCONFIG_DIR: path.join(config.baseDir(), 'data/collectd/collectd.conf.d'),
 
@@ -19,10 +22,11 @@ exports = module.exports = {
     BOX_DATA_DIR: path.join(config.baseDir(), 'data/box'),
     // this is not part of appdata because an icon may be set before install
     APPICONS_DIR: path.join(config.baseDir(), 'data/box/appicons'),
+    APP_CERTS_DIR: path.join(config.baseDir(), 'data/box/certs'),
     MAIL_DATA_DIR: path.join(config.baseDir(), 'data/box/mail'),
 
     CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'data/box/avatar.png'),
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
 
-    FAVICON_FILE: path.join(__dirname + '/../assets/favicon.ico')
+    UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json')
 };

src/routes/apps.js

@@ -43,6 +43,7 @@ function removeInternalAppFields(app) {
         health: app.health,
         location: app.location,
         accessRestriction: app.accessRestriction,
+        oauthProxy: app.oauthProxy,
         lastBackupId: app.lastBackupId,
         manifest: app.manifest,
         portBindings: app.portBindings,
@@ -113,20 +114,28 @@ function installApp(req, res, next) {
     if (typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId is required'));
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
     if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'string') return next(new HttpError(400, 'accessRestriction is required'));
+    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
+    if (typeof data.oauthProxy !== 'boolean') return next(new HttpError(400, 'oauthProxy must be a boolean'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
+    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
+    if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
 
     // allow tests to provide an appId for testing
     var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
 
-    debug('Installing app id:%s storeid:%s loc:%s port:%j restrict:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.manifest);
+    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j oauthproxy:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.oauthProxy, data.manifest);
 
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, function (error) {
+    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.oauthProxy, data.icon || null, data.cert || null, data.key || null, function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
+        if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
+        if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, { id: appId } ));
@@ -149,17 +158,23 @@ function configureApp(req, res, next) {
     if (!data) return next(new HttpError(400, 'Cannot parse data field'));
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
     if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'string') return next(new HttpError(400, 'accessRestriction is required'));
+    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
+    if (typeof data.oauthProxy !== 'boolean') return next(new HttpError(400, 'oauthProxy must be a boolean'));
+    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
+    if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
 
-    debug('Configuring app id:%s location:%s bindings:%j', req.params.id, data.location, data.portBindings);
+    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j oauthProxy:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.oauthProxy);
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, function (error) {
+    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.oauthProxy, data.cert || null, data.key || null, function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, { }));
@@ -253,7 +268,7 @@ function updateApp(req, res, next) {
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
-    debug('Update app id:%s to manifest:%j', req.params.id, data.manifest);
+    debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
 
     apps.update(req.params.id, data.force || false, data.manifest, data.portBindings, data.icon, function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
@@ -272,14 +287,14 @@ function getLogStream(req, res, next) {
 
     debug('Getting logstream of app id:%s', req.params.id);
 
-    var fromLine = req.query.fromLine ? parseInt(req.query.fromLine, 10) : -10; // we ignore last-event-id
-    if (isNaN(fromLine)) return next(new HttpError(400, 'fromLine must be a valid number'));
+    var lines = req.query.lines ? parseInt(req.query.lines, 10) : -10; // we ignore last-event-id
+    if (isNaN(lines)) return next(new HttpError(400, 'lines must be a valid number'));
 
     function sse(id, data) { return 'id: ' + id + '\ndata: ' + data + '\n\n'; }
 
     if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
 
-    apps.getLogStream(req.params.id, fromLine, function (error, logStream) {
+    apps.getLogs(req.params.id, lines, true /* follow */, function (error, logStream) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
@@ -295,7 +310,7 @@ function getLogStream(req, res, next) {
         res.on('close', logStream.close);
         logStream.on('data', function (data) {
             var obj = JSON.parse(data);
-            res.write(sse(obj.lineNumber, JSON.stringify(obj)));
+            res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id
         });
         logStream.on('end', res.end.bind(res));
         logStream.on('error', res.end.bind(res, null));
@@ -305,9 +320,12 @@ function getLogStream(req, res, next) {
 function getLogs(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
+    var lines = req.query.lines ? parseInt(req.query.lines, 10) : 100;
+    if (isNaN(lines)) return next(new HttpError(400, 'lines must be a number'));
+
     debug('Getting logs of app id:%s', req.params.id);
 
-    apps.getLogs(req.params.id, function (error, logStream) {
+    apps.getLogs(req.params.id, lines, false /* follow */, function (error, logStream) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(412, error.message));
         if (error) return next(new HttpError(500, error));
@@ -351,6 +369,8 @@ function exec(req, res, next) {
 
         duplexStream.pipe(res.socket);
         res.socket.pipe(duplexStream);
+
+        res.on('close', duplexStream.close);
     });
 }
 

src/routes/clients.js

@@ -5,7 +5,6 @@
 exports = module.exports = {
     add: add,
     get: get,
-    update: update,
     del: del,
     getAllByUserId: getAllByUserId,
     getClientTokens: getClientTokens,
@@ -13,12 +12,13 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    validUrl = require('valid-url'),
+    clientdb = require('../clientdb.js'),
     clients = require('../clients.js'),
     ClientsError = clients.ClientsError,
     DatabaseError = require('../databaseerror.js'),
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess;
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    validUrl = require('valid-url');
 
 function add(req, res, next) {
     var data = req.body;
@@ -29,10 +29,7 @@ function add(req, res, next) {
     if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
     if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
 
-    // prefix as this route only allows external apps for developers
-    var appId = 'external-' + data.appId;
-
-    clients.add(appId, data.redirectURI, data.scope, function (error, result) {
+    clients.add(data.appId, clientdb.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
         if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, 'Invalid scope'));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(201, result));
@@ -49,22 +46,6 @@ function get(req, res, next) {
     });
 }
 
-function update(req, res, next) {
-    assert.strictEqual(typeof req.params.clientId, 'string');
-
-    var data = req.body;
-
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.appId !== 'string' || !data.appId) return next(new HttpError(400, 'appId is required'));
-    if (typeof data.redirectURI !== 'string' || !data.redirectURI) return next(new HttpError(400, 'redirectURI is required'));
-    if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
-
-    clients.update(req.params.clientId, data.appId, data.redirectURI, function (error, result) {
-        if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(202, result));
-    });
-}
-
 function del(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 

src/routes/cloudron.js

@@ -11,7 +11,6 @@ exports = module.exports = {
     getConfig: getConfig,
     update: update,
     migrate: migrate,
-    setCertificate: setCertificate,
     feedback: feedback
 };
 
@@ -25,7 +24,6 @@ var assert = require('assert'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     superagent = require('superagent'),
-    safe = require('safetydance'),
     updateChecker = require('../updatechecker.js');
 
 /**
@@ -44,22 +42,19 @@ function activate(req, res, next) {
     if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be string'));
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be string'));
     if (typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
-    if ('name' in req.body && typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
 
     var username = req.body.username;
     var password = req.body.password;
     var email = req.body.email;
-    var name = req.body.name || null;
 
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
     debug('activate: username:%s ip:%s', username, ip);
 
-    cloudron.activate(username, password, email, name, ip, function (error, info) {
+    cloudron.activate(username, password, email, ip, function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
         if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
         if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
         if (error && error.reason === CloudronError.BAD_EMAIL) return next(new HttpError(400, 'Bad email'));
-        if (error && error.reason === CloudronError.BAD_NAME) return next(new HttpError(400, 'Bad name'));
         if (error) return next(new HttpError(500, error));
 
         // Now let the api server know we got activated
@@ -143,23 +138,6 @@ function migrate(req, res, next) {
     });
 }
 
-function setCertificate(req, res, next) {
-    assert.strictEqual(typeof req.files, 'object');
-
-    if (!req.files.certificate) return next(new HttpError(400, 'certificate must be provided'));
-    var certificate = safe.fs.readFileSync(req.files.certificate.path, 'utf8');
-
-    if (!req.files.key) return next(new HttpError(400, 'key must be provided'));
-    var key = safe.fs.readFileSync(req.files.key.path, 'utf8');
-
-    cloudron.setCertificate(certificate, key, function (error) {
-        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
-        if (error) return next(new HttpError(500, error));
-
-        next(new HttpSuccess(202, {}));
-    });
-}
-
 function feedback(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 

src/routes/internal.js

@@ -7,6 +7,7 @@ exports = module.exports = {
 };
 
 var cloudron = require('../cloudron.js'),
+    CloudronError = require('../cloudron.js').CloudronError,
     debug = require('debug')('box:routes/internal'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
@@ -14,10 +15,12 @@ var cloudron = require('../cloudron.js'),
 function backup(req, res, next) {
     debug('trigger backup');
 
+    // note that cloudron.backup only waits for backup initiation and not for backup to complete
+    // backup progress can be checked up ny polling the progress api call
     cloudron.backup(function (error) {
-        if (error) debug('Internal route backup failed', error);
-    });
+        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error) return next(new HttpError(500, error));
 
-    // we always succeed to trigger a backup
-    next(new HttpSuccess(202, {}));
+        next(new HttpSuccess(202, {}));
+    });
 }

src/routes/oauth2.js

@@ -3,6 +3,7 @@
 'use strict';
 
 var assert = require('assert'),
+    apps = require('../apps'),
     authcodedb = require('../authcodedb'),
     clientdb = require('../clientdb'),
     config = require('../config.js'),
@@ -27,34 +28,21 @@ var assert = require('assert'),
 // create OAuth 2.0 server
 var gServer = oauth2orize.createServer();
 
+
 // Register serialialization and deserialization functions.
 //
-// When a client redirects a user to user authorization endpoint, an
-// authorization transaction is initiated.  To complete the transaction, the
-// user must authenticate and approve the authorization request.  Because this
-// may involve multiple HTTP request/response exchanges, the transaction is
-// stored in the session.
-//
-// An application must supply serialization functions, which determine how the
-// client object is serialized into the session.  Typically this will be a
-// simple matter of serializing the client's ID, and deserializing by finding
-// the client by ID from the database.
+// The client id is stored in the session and can thus be retrieved for each
+// step in the oauth flow transaction, which involves multiple http requests.
 
 gServer.serializeClient(function (client, callback) {
-    debug('server serialize:', client);
-
     return callback(null, client.id);
 });
 
 gServer.deserializeClient(function (id, callback) {
-    debug('server deserialize:', id);
-
-    clientdb.get(id, function (error, client) {
-        if (error) { return callback(error); }
-        return callback(null, client);
-    });
+    clientdb.get(id, callback);
 });
 
+
 // Register supported grant types.
 
 // Grant authorization codes.  The callback takes the `client` requesting
@@ -64,21 +52,17 @@ gServer.deserializeClient(function (id, callback) {
 // the application.  The application issues a code, which is bound to these
 // values, and will be exchanged for an access token.
 
-// we use , (comma) as scope separator
 gServer.grant(oauth2orize.grant.code({ scopeSeparator: ',' }, function (client, redirectURI, user, ares, callback) {
-    debug('grant code:', client, redirectURI, user.id, ares);
+    debug('grant code:', client.id, redirectURI, user.id, ares);
 
     var code = hat(256);
     var expiresAt = Date.now() + 60 * 60000; // 1 hour
-    var scopes = client.scope ? client.scope.split(',') : ['profile','roleUser'];
-
-    if (scopes.indexOf('roleAdmin') !== -1 && !user.admin) {
-        debug('grant code: not allowed, you need to be admin');
-        return callback(new Error('Admin capabilities required'));
-    }
 
     authcodedb.add(code, client.id, user.username, expiresAt, function (error) {
         if (error) return callback(error);
+
+        debug('grant code: new auth code for client %s code %s', client.id, code);
+
         callback(null, code);
     });
 }));
@@ -93,7 +77,7 @@ gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client,
     tokendb.add(token, tokendb.PREFIX_USER + user.id, client.id, expires, client.scope, function (error) {
         if (error) return callback(error);
 
-        debug('new access token for client ' + client.id + ' token ' + token);
+        debug('grant token: new access token for client %s token %s', client.id, token);
 
         callback(null, token);
     });
@@ -123,7 +107,7 @@ gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI,
             tokendb.add(token, tokendb.PREFIX_USER + authCode.userId, authCode.clientId, expires, client.scope, function (error) {
                 if (error) return callback(error);
 
-                debug('new access token for client ' + client.id + ' token ' + token);
+                debug('exchange: new access token for client %s token %s', client.id, token);
 
                 callback(null, token);
             });
@@ -148,93 +132,89 @@ session.ensureLoggedIn = function (redirectTo) {
     };
 };
 
+function renderTemplate(res, template, data) {
+    assert.strictEqual(typeof res, 'object');
+    assert.strictEqual(typeof template, 'string');
+    assert.strictEqual(typeof data, 'object');
+
+    res.render(template, data);
+}
+
 function sendErrorPageOrRedirect(req, res, message) {
     assert.strictEqual(typeof req, 'object');
     assert.strictEqual(typeof res, 'object');
     assert.strictEqual(typeof message, 'string');
 
-    debug('sendErrorPageOrRedirect: returnTo "%s".', req.query.returnTo, message);
+    debug('sendErrorPageOrRedirect: returnTo %s.', req.query.returnTo, message);
 
     if (typeof req.query.returnTo !== 'string') {
-        res.render('error', {
+        renderTemplate(res, 'error', {
             adminOrigin: config.adminOrigin(),
             message: message
         });
     } else {
         var u = url.parse(req.query.returnTo);
-        if (!u.protocol || !u.host) return res.render('error', {
-            adminOrigin: config.adminOrigin(),
-            message: 'Invalid request. returnTo query is not a valid URI. ' + message
-        });
+        if (!u.protocol || !u.host) {
+            return renderTemplate(res, 'error', {
+                adminOrigin: config.adminOrigin(),
+                message: 'Invalid request. returnTo query is not a valid URI. ' + message
+            });
+        }
 
         res.redirect(util.format('%s//%s', u.protocol, u.host));
     }
 }
 
+// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
+// This usually happens when the OAuth client ID is wrong
 function sendError(req, res, message) {
     assert.strictEqual(typeof req, 'object');
     assert.strictEqual(typeof res, 'object');
     assert.strictEqual(typeof message, 'string');
 
-    res.render('error', {
+    renderTemplate(res, 'error', {
         adminOrigin: config.adminOrigin(),
         message: message
     });
 }
 
-// Main login form username and password
+// -> GET /api/v1/session/login
 function loginForm(req, res) {
     if (typeof req.session.returnTo !== 'string') return sendErrorPageOrRedirect(req, res, 'Invalid login request. No returnTo provided.');
 
     var u = url.parse(req.session.returnTo, true);
     if (!u.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid login request. No client_id provided.');
 
-    var cloudronName = '';
-
     function render(applicationName, applicationLogo) {
-        res.render('login', {
+        renderTemplate(res, 'login', {
             adminOrigin: config.adminOrigin(),
             csrf: req.csrfToken(),
-            cloudronName: cloudronName,
             applicationName: applicationName,
             applicationLogo: applicationLogo,
             error: req.query.error || null
         });
     }
 
-    settings.getCloudronName(function (error, name) {
-        if (error) return sendError(req, res, 'Internal Error');
+    clientdb.get(u.query.client_id, function (error, result) {
+        if (error) return sendError(req, res, 'Unknown OAuth client');
 
-        cloudronName = name;
+        switch (result.type) {
+            case clientdb.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
+            case clientdb.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
+            case clientdb.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
+            default: break;
+        }
 
-        clientdb.get(u.query.client_id, function (error, result) {
-            if (error) return sendError(req, res, 'Unknown OAuth client');
+        appdb.get(result.appId, function (error, result) {
+            if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
 
-            // Handle our different types of oauth clients
-            var appId = result.appId;
-            if (appId === constants.ADMIN_CLIENT_ID) {
-                return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
-            } else if (appId === constants.TEST_CLIENT_ID) {
-                return render(constants.TEST_NAME, '/api/v1/cloudron/avatar');
-            } else if (appId.indexOf('external-') === 0) {
-                return render('External Application', '/api/v1/cloudron/avatar');
-            } else if (appId.indexOf('addon-') === 0) {
-                appId = appId.slice('addon-'.length);
-            } else if (appId.indexOf('proxy-') === 0) {
-                appId = appId.slice('proxy-'.length);
-            }
-
-            appdb.get(appId, function (error, result) {
-                if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
-
-                var applicationName = result.location || config.fqdn();
-                render(applicationName, '/api/v1/cloudron/avatar');
-            });
+            var applicationName = result.location || config.fqdn();
+            render(applicationName, '/api/v1/apps/' + result.id + '/icon');
         });
     });
 }
 
-// performs the login POST from the login form
+// -> POST /api/v1/session/login
 function login(req, res) {
     var returnTo = req.session.returnTo || req.query.returnTo;
 
@@ -246,7 +226,7 @@ function login(req, res) {
     });
 }
 
-// ends the current session
+// -> GET /api/v1/session/logout
 function logout(req, res) {
     req.logout();
 
@@ -257,7 +237,7 @@ function logout(req, res) {
 // Form to enter email address to send a password reset request mail
 // -> GET /api/v1/session/password/resetRequest.html
 function passwordResetRequestSite(req, res) {
-    res.render('password_reset_request', { adminOrigin: config.adminOrigin(), csrf: req.csrfToken() });
+    renderTemplate(res, 'password_reset_request', { adminOrigin: config.adminOrigin(), csrf: req.csrfToken() });
 }
 
 // This route is used for above form submission
@@ -281,19 +261,22 @@ function passwordResetRequest(req, res, next) {
 
 // -> GET /api/v1/session/password/sent.html
 function passwordSentSite(req, res) {
-    res.render('password_reset_sent', { adminOrigin: config.adminOrigin() });
+    renderTemplate(res, 'password_reset_sent', { adminOrigin: config.adminOrigin() });
 }
 
 // -> GET /api/v1/session/password/setup.html
 function passwordSetupSite(req, res, next) {
     if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
 
-    debug('passwordSetupSite: with token %s.', req.query.reset_token);
-
     user.getByResetToken(req.query.reset_token, function (error, user) {
         if (error) return next(new HttpError(401, 'Invalid reset_token'));
 
-        res.render('password_setup', { adminOrigin: config.adminOrigin(), user: user, csrf: req.csrfToken(), resetToken: req.query.reset_token });
+        renderTemplate(res, 'password_setup', {
+            adminOrigin: config.adminOrigin(),
+            user: user,
+            csrf: req.csrfToken(),
+            resetToken: req.query.reset_token
+        });
     });
 }
 
@@ -301,12 +284,15 @@ function passwordSetupSite(req, res, next) {
 function passwordResetSite(req, res, next) {
     if (!req.query.reset_token) return next(new HttpError(400, 'Missing reset_token'));
 
-    debug('passwordResetSite: with token %s.', req.query.reset_token);
-
     user.getByResetToken(req.query.reset_token, function (error, user) {
         if (error) return next(new HttpError(401, 'Invalid reset_token'));
 
-        res.render('password_reset', { adminOrigin: config.adminOrigin(), user: user, csrf: req.csrfToken(), resetToken: req.query.reset_token });
+        renderTemplate(res, 'password_reset', {
+            adminOrigin: config.adminOrigin(),
+            user: user,
+            csrf: req.csrfToken(),
+            resetToken: req.query.reset_token
+        });
     });
 }
 
@@ -332,50 +318,28 @@ function passwordReset(req, res, next) {
 }
 
 
-/*
-
-  The callback page takes the redirectURI and the authCode and redirects the browser accordingly
-
-*/
+// The callback page takes the redirectURI and the authCode and redirects the browser accordingly
+//
+// -> GET /api/v1/session/callback
 var callback = [
     session.ensureLoggedIn('/api/v1/session/login'),
     function (req, res) {
-        debug('callback: with callback server ' + req.query.redirectURI);
-        res.render('callback', { adminOrigin: config.adminOrigin(), callbackServer: req.query.redirectURI });
+        renderTemplate(res, 'callback', { adminOrigin: config.adminOrigin(), callbackServer: req.query.redirectURI });
     }
 ];
 
 
-/*
-
-  This indicates a missing OAuth client session or invalid client ID
-
-*/
-var error = [
-    session.ensureLoggedIn('/api/v1/session/login'),
-    function (req, res) {
-        sendErrorPageOrRedirect(req, res, 'Invalid OAuth Client');
-    }
-];
-
-
-/*
-
-  The authorization endpoint is the entry point for an OAuth login.
-
-  Each app would start OAuth by redirecting the user to:
-
-    /api/v1/oauth/dialog/authorize?response_type=code&client_id=<clientId>&redirect_uri=<callbackURL>&scope=<ignored>
-
-  - First, this will ensure the user is logged in.
-  - Then in normal OAuth it would ask the user for permissions to the scopes, which we will do on app installation
-  - Then it will redirect the browser to the given <callbackURL> containing the authcode in the query
-
-  Scopes are set by the app during installation, the ones given on OAuth transaction start are simply ignored.
-
-*/
+// The authorization endpoint is the entry point for an OAuth login.
+//
+// Each app would start OAuth by redirecting the user to:
+//
+//    /api/v1/oauth/dialog/authorize?response_type=code&client_id=<clientId>&redirect_uri=<callbackURL>&scope=<ignored>
+//
+//  - First, this will ensure the user is logged in.
+//  - Then it will redirect the browser to the given <callbackURL> containing the authcode in the query
+//
+// -> GET /api/v1/oauth/dialog/authorize
 var authorization = [
-    // extract the returnTo origin and set as query param
     function (req, res, next) {
         if (!req.query.redirect_uri) return sendErrorPageOrRedirect(req, res, 'Invalid request. redirect_uri query param is not set.');
         if (!req.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid request. client_id query param is not set.');
@@ -384,10 +348,10 @@ var authorization = [
 
         session.ensureLoggedIn('/api/v1/session/login?returnTo=' + req.query.redirect_uri)(req, res, next);
     },
-    gServer.authorization(function (clientID, redirectURI, callback) {
-        debug('authorization: client %s with callback to %s.', clientID, redirectURI);
+    gServer.authorization({}, function (clientId, redirectURI, callback) {
+        debug('authorization: client %s with callback to %s.', clientId, redirectURI);
 
-        clientdb.get(clientID, function (error, client) {
+        clientdb.get(clientId, function (error, client) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
@@ -395,50 +359,36 @@ var authorization = [
             var redirectPath = url.parse(redirectURI).path;
             var redirectOrigin = client.redirectURI;
 
-            callback(null, client, '/api/v1/session/callback?redirectURI=' + url.resolve(redirectOrigin, redirectPath));
+            callback(null, client, '/api/v1/session/callback?redirectURI=' + encodeURIComponent(url.resolve(redirectOrigin, redirectPath)));
         });
     }),
-    // Until we have OAuth scopes, skip decision dialog
-    // OAuth sopes skip START
     function (req, res, next) {
-        assert.strictEqual(typeof req.body, 'object');
-        assert.strictEqual(typeof req.oauth2, 'object');
+        // Handle our different types of oauth clients
+        var type = req.oauth2.client.type;
 
-        var scopes = req.oauth2.client.scope ? req.oauth2.client.scope.split(',') : ['profile','roleUser'];
+        if (type === clientdb.TYPE_ADMIN) return next();
+        if (type === clientdb.TYPE_EXTERNAL) return next();
+        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unkonwn OAuth client.');
 
-        if (scopes.indexOf('roleAdmin') !== -1 && !req.user.admin) {
-            return sendErrorPageOrRedirect(req, res, 'Admin capabilities required');
-        }
+        appdb.get(req.oauth2.client.appId, function (error, appObject) {
+            if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
 
-        req.body.transaction_id = req.oauth2.transactionID;
-        next();
+            if (!apps.hasAccessTo(appObject, req.oauth2.user)) return sendErrorPageOrRedirect(req, res, 'No access to this app.');
+
+            next();
+        });
     },
-    gServer.decision(function(req, done) {
-        debug('decision: with scope', req.oauth2.req.scope);
-        return done(null, { scope: req.oauth2.req.scope });
-    })
-    // OAuth sopes skip END
-    // function (req, res) {
-    //     res.render('dialog', { transactionID: req.oauth2.transactionID, user: req.user, client: req.oauth2.client, csrf: req.csrfToken() });
-    // }
-];
-
-// this triggers the above grant middleware and handles the user's decision if he accepts the access
-var decision = [
-    session.ensureLoggedIn('/api/v1/session/login'),
-    gServer.decision()
+    gServer.decision({ loadTransaction: false })
 ];
 
 
-/*
-
-  The token endpoint allows an OAuth client to exchange an authcode with an accesstoken.
-
-  Authcodes are obtained using the authorization endpoint. The route is authenticated by
-  providing a Basic auth with clientID as username and clientSecret as password.
-  An authcode is only good for one such exchange to an accesstoken.
-
-*/
+//  The token endpoint allows an OAuth client to exchange an authcode with an accesstoken.
+//
+//  Authcodes are obtained using the authorization endpoint. The route is authenticated by
+//  providing a Basic auth with clientID as username and clientSecret as password.
+//  An authcode is only good for one such exchange to an accesstoken.
+//
+// -> POST /api/v1/oauth/token
 var token = [
     passport.authenticate(['basic', 'oauth2-client-password'], { session: false }),
     gServer.token(),
@@ -446,18 +396,15 @@ var token = [
 ];
 
 
-/*
-
-  The scope middleware provides an auth middleware for routes.
-
-  It is used for API routes, which are authenticated using accesstokens.
-  Those accesstokens carry OAuth scopes and the middleware takes the required
-  scope as an argument and will verify the accesstoken against it.
-
-  See server.js:
-    var profileScope = routes.oauth2.scope('profile');
-
-*/
+//  The scope middleware provides an auth middleware for routes.
+//
+//  It is used for API routes, which are authenticated using accesstokens.
+//  Those accesstokens carry OAuth scopes and the middleware takes the required
+//  scope as an argument and will verify the accesstoken against it.
+//
+//  See server.js:
+//    var profileScope = routes.oauth2.scope('profile');
+//
 function scope(requestedScope) {
     assert.strictEqual(typeof requestedScope, 'string');
 
@@ -499,7 +446,6 @@ exports = module.exports = {
     login: login,
     logout: logout,
     callback: callback,
-    error: error,
     passwordResetRequestSite: passwordResetRequestSite,
     passwordResetRequest: passwordResetRequest,
     passwordSentSite: passwordSentSite,
@@ -507,7 +453,6 @@ exports = module.exports = {
     passwordSetupSite: passwordSetupSite,
     passwordReset: passwordReset,
     authorization: authorization,
-    decision: decision,
     token: token,
     scope: scope,
     csrf: csrf

src/routes/settings.js

@@ -10,7 +10,16 @@ exports = module.exports = {
     setCloudronName: setCloudronName,
 
     getCloudronAvatar: getCloudronAvatar,
-    setCloudronAvatar: setCloudronAvatar
+    setCloudronAvatar: setCloudronAvatar,
+
+    getDnsConfig: getDnsConfig,
+    setDnsConfig: setDnsConfig,
+
+    getBackupConfig: getBackupConfig,
+    setBackupConfig: setBackupConfig,
+
+    setCertificate: setCertificate,
+    setAdminCertificate: setAdminCertificate
 };
 
 var assert = require('assert'),
@@ -83,3 +92,75 @@ function getCloudronAvatar(req, res, next) {
         res.status(200).send(avatar);
     });
 }
+
+function getDnsConfig(req, res, next) {
+    settings.getDnsConfig(function (error, config) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, config));
+    });
+}
+
+function setDnsConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+
+    settings.setDnsConfig(req.body, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+function getBackupConfig(req, res, next) {
+    settings.getBackupConfig(function (error, config) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, config));
+    });
+}
+
+function setBackupConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+
+    settings.setBackupConfig(req.body, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+// default fallback cert
+function setCertificate(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.cert || typeof req.body.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (!req.body.key || typeof req.body.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    settings.setCertificate(req.body.cert, req.body.key, function (error) {
+        if (error && error.reason === SettingsError.INVALID_CERT) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}
+
+// only webadmin cert, until it can be treated just like a normal app
+function setAdminCertificate(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.cert || typeof req.body.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (!req.body.key || typeof req.body.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    settings.setAdminCertificate(req.body.cert, req.body.key, function (error) {
+        if (error && error.reason === SettingsError.INVALID_CERT) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(202, {}));
+    });
+}

src/routes/test/apps-test.js

@@ -16,22 +16,21 @@ var appdb = require('../../appdb.js'),
     config = require('../../config.js'),
     constants = require('../../constants.js'),
     database = require('../../database.js'),
-    docker = require('../../docker.js'),
+    docker = require('../../docker.js').connection,
     expect = require('expect.js'),
     fs = require('fs'),
     hock = require('hock'),
     http = require('http'),
     https = require('https'),
+    js2xml = require('js2xmlparser'),
     net = require('net'),
     nock = require('nock'),
-    os = require('os'),
     paths = require('../../paths.js'),
     redis = require('redis'),
     request = require('superagent'),
     safe = require('safetydance'),
     server = require('../../server.js'),
     settings = require('../../settings.js'),
-    sysinfo = require('../../sysinfo.js'),
     tokendb = require('../../tokendb.js'),
     url = require('url'),
     util = require('util'),
@@ -40,17 +39,43 @@ var appdb = require('../../appdb.js'),
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
+// Test image information
+var TEST_IMAGE_REPO = 'cloudron/test';
+var TEST_IMAGE_TAG = '10.0.0';
+var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG).toString('utf8').trim();
+
 var APP_STORE_ID = 'test', APP_ID;
 var APP_LOCATION = 'appslocation';
 var APP_LOCATION_2 = 'appslocationtwo';
 var APP_LOCATION_NEW = 'appslocationnew';
+
 var APP_MANIFEST = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-app/CloudronManifest.json', 'utf8'));
-APP_MANIFEST.dockerImage = 'girish/test:0.2.0';
+APP_MANIFEST.dockerImage = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
+
+var APP_MANIFEST_1 = JSON.parse(fs.readFileSync(__dirname + '/../../../../test-app/CloudronManifest.json', 'utf8'));
+APP_MANIFEST_1.dockerImage = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
+APP_MANIFEST_1.singleUser = true;
+
 var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='admin@me.com';
 var USERNAME_1 = 'user', PASSWORD_1 = 'password', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;
 
+var awsHostedZones = {
+     HostedZones: [{
+         Id: '/hostedzone/ZONEID',
+         Name: 'localhost.',
+         CallerReference: '305AFD59-9D73-4502-B020-F4E6F889CB30',
+         ResourceRecordSetCount: 2,
+         ChangeInfo: {
+             Id: '/change/CKRTFJA0ANHXB',
+             Status: 'INSYNC'
+         }
+     }],
+    IsTruncated: false,
+    MaxItems: '100'
+ };
+
 function startDockerProxy(interceptor, callback) {
     assert.strictEqual(typeof interceptor, 'function');
 
@@ -79,10 +104,12 @@ function startDockerProxy(interceptor, callback) {
 
 function setup(done) {
     async.series([
-        server.start.bind(server),
-
+        // first clear, then start server. otherwise, taskmanager spins up tasks for obsolete appIds
+        database.initialize,
         database._clear,
 
+        server.start.bind(server),
+
         function (callback) {
             var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
             var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
@@ -104,11 +131,11 @@ function setup(done) {
             });
         },
 
-        child_process.exec.bind(null, __dirname + '/start_addons.sh'),
-
         function (callback) {
-            callback(null);
+            console.log('Starting addons, this can take 10 seconds');
+            child_process.exec(__dirname + '/start_addons.sh', callback);
         },
+
         function (callback) {
             request.post(SERVER_URL + '/api/v1/users')
                    .query({ access_token: token })
@@ -119,22 +146,28 @@ function setup(done) {
 
                 callback(null);
             });
-        }, function (callback) {
+        },
+
+        function (callback) {
             token_1 = tokendb.generateToken();
 
             // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
             tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
-        }
+        },
+
+        settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
+        settings.setBackupConfig.bind(null, { provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
     ], done);
 }
 
 function cleanup(done) {
+    // db is not cleaned up here since it's too late to call it after server.stop. if called before server.stop taskmanager apptasks are unhappy :/
     async.series([
-        database._clear,
-
         server.stop,
 
-        child_process.exec.bind(null, 'docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb')
+        function (callback) { setTimeout(callback, 2000); }, // give taskmanager tasks couple of seconds to finish
+
+        child_process.exec.bind(null, 'docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb; docker rm -f mail')
     ], done);
 }
 
@@ -143,10 +176,19 @@ describe('App API', function () {
     var dockerProxy;
 
     before(function (done) {
-        dockerProxy = startDockerProxy(function interceptor() { return false; }, function () {
+        dockerProxy = startDockerProxy(function interceptor(req, res) {
+            if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE_ID + '?force=true&noprune=false') {
+                res.writeHead(200);
+                res.end();
+                return true;
+            }
+
+            return false;
+        }, function () {
             setup(done);
         });
     });
+
     after(function (done) {
         APP_ID = null;
         cleanup(function () {
@@ -189,7 +231,7 @@ describe('App API', function () {
     it('app install fails - invalid location', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             expect(res.body.message).to.eql('Hostname can only contain alphanumerics and hyphen');
@@ -200,7 +242,7 @@ describe('App API', function () {
     it('app install fails - invalid location type', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             expect(res.body.message).to.eql('location is required');
@@ -211,7 +253,7 @@ describe('App API', function () {
     it('app install fails - reserved admin location', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
@@ -222,7 +264,7 @@ describe('App API', function () {
     it('app install fails - reserved api location', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null, oauthProxy: true })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             expect(res.body.message).to.eql(constants.API_LOCATION + ' is reserved');
@@ -233,7 +275,7 @@ describe('App API', function () {
     it('app install fails - portBindings must be object', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             expect(res.body.message).to.eql('portBindings must be an object');
@@ -244,7 +286,7 @@ describe('App API', function () {
     it('app install fails - accessRestriction is required', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             expect(res.body.message).to.eql('accessRestriction is required');
@@ -252,10 +294,54 @@ describe('App API', function () {
         });
     });
 
+    it('app install fails - accessRestriction type is wrong', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/install')
+               .query({ access_token: token })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '', oauthProxy: false })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            expect(res.body.message).to.eql('accessRestriction is required');
+            done(err);
+        });
+    });
+
+    it('app install fails - accessRestriction no users not allowed', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/install')
+               .query({ access_token: token })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null, oauthProxy: false })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            expect(res.body.message).to.eql('accessRestriction must specify one user');
+            done(err);
+        });
+    });
+
+    it('app install fails - accessRestriction too many users not allowed', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/install')
+               .query({ access_token: token })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] }, oauthProxy: false })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            expect(res.body.message).to.eql('accessRestriction must specify one user');
+            done(err);
+        });
+    });
+
+    it('app install fails - oauthProxy is required', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/install')
+               .query({ access_token: token })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            expect(res.body.message).to.eql('oauthProxy must be a boolean');
+            done(err);
+        });
+    });
+
     it('app install fails for non admin', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token_1 })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(403);
             done(err);
@@ -267,7 +353,7 @@ describe('App API', function () {
 
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(402);
             expect(fake.isDone()).to.be.ok();
@@ -280,7 +366,7 @@ describe('App API', function () {
 
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(202);
             expect(res.body.id).to.be.a('string');
@@ -295,7 +381,7 @@ describe('App API', function () {
 
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(409);
             expect(fake.isDone()).to.be.ok();
@@ -419,7 +505,7 @@ describe('App API', function () {
 
         request.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: '' })
+               .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null, oauthProxy: false })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(202);
             expect(res.body.id).to.be.a('string');
@@ -448,7 +534,7 @@ describe('App API', function () {
 
                 request.post(SERVER_URL + '/api/v1/apps/install')
                        .query({ access_token: token })
-                       .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: '' })
+                       .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null, oauthProxy: false })
                        .end(function (err, res) {
                     expect(res.statusCode).to.equal(202);
                     expect(res.body.id).to.be.a('string');
@@ -473,7 +559,8 @@ describe('App API', function () {
 describe('App installation', function () {
     this.timeout(50000);
 
-    var hockInstance = hock.createHock({ throwOnUnmatched: false }), hockServer, dockerProxy;
+    var apiHockInstance = hock.createHock({ throwOnUnmatched: false }), apiHockServer, dockerProxy;
+    var awsHockInstance = hock.createHock({ throwOnUnmatched: false }), awsHockServer;
     var imageDeleted = false, imageCreated = false;
 
     before(function (done) {
@@ -482,12 +569,12 @@ describe('App installation', function () {
         async.series([
             function (callback) {
                 dockerProxy = startDockerProxy(function interceptor(req, res) {
-                    if (req.method === 'POST' && req.url === '/images/create?fromImage=girish%2Ftest&tag=0.2.0') {
+                    if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
                         imageCreated = true;
                         res.writeHead(200);
                         res.end();
                         return true;
-                    } else if (req.method === 'DELETE' && req.url === '/images/c7ddfc8fb7cd8a14d4d70153a199ff0c6e9b709807aeec5a7b799d60618731d1?force=true&noprune=false') {
+                    } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE_ID + '?force=true&noprune=false') {
                         imageDeleted = true;
                         res.writeHead(200);
                         res.end();
@@ -500,28 +587,38 @@ describe('App installation', function () {
             setup,
 
             function (callback) {
-                hockInstance
+                apiHockInstance
                     .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
-                    .replyWithFile(200, path.resolve(__dirname, '../../../webadmin/src/img/appicon_fallback.png'))
-                    .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP_LOCATION, type: 'A', value: sysinfo.getIp() } ] })
-                    .reply(201, { ids: [ 'dnsrecordid' ] }, { 'Content-Type': 'application/json' })
-                    .delete('/api/v1/subdomains/dnsrecordid?token=' + config.token())
-                    .reply(204, { }, { 'Content-Type': 'application/json' });
+                    .replyWithFile(200, path.resolve(__dirname, '../../../webadmin/src/img/appicon_fallback.png'));
 
                 var port = parseInt(url.parse(config.apiServerOrigin()).port, 10);
-                hockServer = http.createServer(hockInstance.handler).listen(port, callback);
+                apiHockServer = http.createServer(apiHockInstance.handler).listen(port, callback);
+            },
+
+            function (callback) {
+                awsHockInstance
+                    .get('/2013-04-01/hostedzone')
+                    .max(Infinity)
+                    .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }), { 'Content-Type': 'application/xml' })
+                    .filteringRequestBody(function (unusedBody) { return ''; }) // strip out body
+                    .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+                    .max(Infinity)
+                    .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'dnsrecordid', Status: 'INSYNC' } }), { 'Content-Type': 'application/xml' });
+
+                awsHockServer = http.createServer(awsHockInstance.handler).listen(5353, callback);
             }
         ], done);
     });
 
     after(function (done) {
         APP_ID = null;
-        cleanup(function (error) {
-            if (error) return done(error);
-            hockServer.close(function () {
-                dockerProxy.close(done);
-            });
-        });
+
+        async.series([
+            cleanup,
+            apiHockServer.close.bind(apiHockServer),
+            awsHockServer.close.bind(awsHockServer),
+            dockerProxy.close.bind(dockerProxy)
+        ], done);
     });
 
     var appResult = null /* the json response */, appEntry = null /* entry from database */;
@@ -544,7 +641,7 @@ describe('App installation', function () {
 
         request.post(SERVER_URL + '/api/v1/apps/install')
               .query({ access_token: token })
-              .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: '' })
+              .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null, oauthProxy: false })
               .end(function (err, res) {
             expect(res.statusCode).to.equal(202);
             expect(fake.isDone()).to.be.ok();
@@ -576,9 +673,12 @@ describe('App installation', function () {
             expect(data.Config.Env).to.contain('WEBADMIN_ORIGIN=' + config.adminOrigin());
             expect(data.Config.Env).to.contain('API_ORIGIN=' + config.adminOrigin());
             expect(data.Config.Env).to.contain('CLOUDRON=1');
-            clientdb.getByAppId('addon-' + appResult.id, function (error, client) {
+            expect(data.Config.Env).to.contain('APP_ORIGIN=https://' + config.appFqdn(APP_LOCATION));
+            expect(data.Config.Env).to.contain('APP_DOMAIN=' + config.appFqdn(APP_LOCATION));
+            expect(data.Config.Hostname).to.be(APP_LOCATION);
+            clientdb.getByAppIdAndType(appResult.id, clientdb.TYPE_OAUTH, function (error, client) {
                 expect(error).to.not.be.ok();
-                expect(client.id.length).to.be(46); // cid-addon- + 32 hex chars (128 bits) + 4 hyphens
+                expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
                 expect(client.clientSecret.length).to.be(64); // 32 hex chars (256 bits)
                 expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
                 expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
@@ -617,7 +717,14 @@ describe('App installation', function () {
     it('installation - running container has volume mounted', function (done) {
         docker.getContainer(appEntry.containerId).inspect(function (error, data) {
             expect(error).to.not.be.ok();
-            expect(data.Volumes['/app/data']).to.eql(paths.DATA_DIR + '/' + APP_ID + '/data');
+
+            // support newer docker versions
+            if (data.Volumes) {
+                expect(data.Volumes['/app/data']).to.eql(paths.DATA_DIR + '/' + APP_ID + '/data');
+            } else {
+                expect(data.Mounts.filter(function (mount) { return mount.Destination === '/app/data'; })[0].Source).to.eql(paths.DATA_DIR + '/' + APP_ID + '/data');
+            }
+
             done();
         });
     });
@@ -654,10 +761,7 @@ describe('App installation', function () {
 
             expect(urlp.hostname).to.be('redis-' + APP_ID);
 
-            var isMac = os.platform() === 'darwin';
-            var client =
-                isMac ? redis.createClient(parseInt(exportedRedisPort, 10), '127.0.0.1', { auth_pass: password })
-                      : redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
+            var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
             client.on('error', done);
             client.set('key', 'value');
             client.get('key', function (err, reply) {
@@ -693,7 +797,7 @@ describe('App installation', function () {
             child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
                 expect(!error).to.be.ok();
                 expect(stdout.length).to.be(0);
-                expect(stderr.length).to.be(0);
+                // expect(stderr.length).to.be(0); // "Warning: Using a password on the command line interface can be insecure."
                 done();
             });
         });
@@ -729,7 +833,15 @@ describe('App installation', function () {
         });
     });
 
-    it('logs - stdout and stderr', function (done) {
+    it('installation - scheduler', function (done) {
+        async.retry({ times: 100, interval: 1000 }, function (retryCallback) {
+            if (fs.existsSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env')) return retryCallback();
+
+            retryCallback(new Error('not run yet'));
+        }, done);
+    });
+
+    xit('logs - stdout and stderr', function (done) {
         request.get(SERVER_URL + '/api/v1/apps/' + APP_ID + '/logs')
             .query({ access_token: token })
             .end(function (err, res) {
@@ -743,7 +855,7 @@ describe('App installation', function () {
         });
     });
 
-    it('logStream - requires event-stream accept header', function (done) {
+    xit('logStream - requires event-stream accept header', function (done) {
         request.get(SERVER_URL + '/api/v1/apps/' + APP_ID + '/logstream')
             .query({ access_token: token, fromLine: 0 })
             .end(function (err, res) {
@@ -753,7 +865,7 @@ describe('App installation', function () {
     });
 
 
-    it('logStream - stream logs', function (done) {
+    xit('logStream - stream logs', function (done) {
         var options = {
             port: config.get('port'), host: 'localhost', path: '/api/v1/apps/' + APP_ID + '/logstream?access_token=' + token,
             headers: { 'Accept': 'text/event-stream', 'Connection': 'keep-alive' }
@@ -882,9 +994,13 @@ describe('App installation', function () {
     });
 
     it('uninstalled - unregistered subdomain', function (done) {
-        hockInstance.done(function (error) { // checks if all the hockServer APIs were called
+        apiHockInstance.done(function (error) { // checks if all the apiHockServer APIs were called
             expect(!error).to.be.ok();
-            done();
+
+            awsHockInstance.done(function (error) {
+                expect(!error).to.be.ok();
+                done();
+            });
         });
     });
 
@@ -904,20 +1020,28 @@ describe('App installation', function () {
 describe('App installation - port bindings', function () {
     this.timeout(50000);
 
-    var hockInstance = hock.createHock({ throwOnUnmatched: false }), hockServer, dockerProxy;
+    var apiHockInstance = hock.createHock({ throwOnUnmatched: false }), apiHockServer, dockerProxy;
+    var awsHockInstance = hock.createHock({ throwOnUnmatched: false }), awsHockServer;
     var imageDeleted = false, imageCreated = false;
 
+    // *.foobar.com
+    var validCert1 = '-----BEGIN CERTIFICATE-----\nMIIBvjCCAWgCCQCg957GWuHtbzANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJE\nRTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xEDAOBgNVBAoMB05l\nYnVsb24xDDAKBgNVBAsMA0NUTzEVMBMGA1UEAwwMKi5mb29iYXIuY29tMB4XDTE1\nMTAyODEzMDI1MFoXDTE2MTAyNzEzMDI1MFowZjELMAkGA1UEBhMCREUxDzANBgNV\nBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRAwDgYDVQQKDAdOZWJ1bG9uMQww\nCgYDVQQLDANDVE8xFTATBgNVBAMMDCouZm9vYmFyLmNvbTBcMA0GCSqGSIb3DQEB\nAQUAA0sAMEgCQQC0FKf07ZWMcABFlZw+GzXK9EiZrlJ1lpnu64RhN99z7MXRr8cF\nnZVgY3jgatuyR5s3WdzUvye2eJ0rNicl2EZJAgMBAAEwDQYJKoZIhvcNAQELBQAD\nQQAw4bteMZAeJWl2wgNLw+wTwAH96E0jyxwreCnT5AxJLmgimyQ0XOF4FsssdRFj\nxD9WA+rktelBodJyPeTDNhIh\n-----END CERTIFICATE-----';
+    var validKey1 = '-----BEGIN RSA PRIVATE KEY-----\nMIIBOQIBAAJBALQUp/TtlYxwAEWVnD4bNcr0SJmuUnWWme7rhGE333PsxdGvxwWd\nlWBjeOBq27JHmzdZ3NS/J7Z4nSs2JyXYRkkCAwEAAQJALV2eykcoC48TonQEPmkg\nbhaIS57syw67jMLsQImQ02UABKzqHPEKLXPOZhZPS9hsC/hGIehwiYCXMUlrl+WF\nAQIhAOntBI6qaecNjAAVG7UbZclMuHROUONmZUF1KNq6VyV5AiEAxRLkfHWy52CM\njOQrX347edZ30f4QczvugXwsyuU9A1ECIGlGZ8Sk4OBA8n6fAUcyO06qnmCJVlHg\npTUeOvKk5c9RAiBs28+8dCNbrbhVhx/yQr9FwNM0+ttJW/yWJ+pyNQhr0QIgJTT6\nxwCWYOtbioyt7B9l+ENy3AMSO3Uq+xmIKkvItK4=\n-----END RSA PRIVATE KEY-----';
+
     before(function (done) {
+        config.set('fqdn', 'test.foobar.com');
+
         APP_ID = uuid.v4();
+
         async.series([
             function (callback) {
                 dockerProxy = startDockerProxy(function interceptor(req, res) {
-                    if (req.method === 'POST' && req.url === '/images/create?fromImage=girish%2Ftest&tag=0.2.0') {
+                    if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
                         imageCreated = true;
                         res.writeHead(200);
                         res.end();
                         return true;
-                    } else if (req.method === 'DELETE' && req.url === '/images/c7ddfc8fb7cd8a14d4d70153a199ff0c6e9b709807aeec5a7b799d60618731d1?force=true&noprune=false') {
+                    } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE_ID + '?force=true&noprune=false') {
                         imageDeleted = true;
                         res.writeHead(200);
                         res.end();
@@ -930,35 +1054,39 @@ describe('App installation - port bindings', function () {
             setup,
 
             function (callback) {
-                hockInstance
-                    // app install
+                apiHockInstance
                     .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
-                    .replyWithFile(200, path.resolve(__dirname, '../../../webadmin/src/img/appicon_fallback.png'))
-                    .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP_LOCATION, type: 'A', value: sysinfo.getIp() } ] })
-                    .reply(201, { ids: [ 'dnsrecordid' ] }, { 'Content-Type': 'application/json' })
-                    // app configure
-                    .delete('/api/v1/subdomains/dnsrecordid?token=' + config.token())
-                    .reply(204, { }, { 'Content-Type': 'application/json' })
-                    .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP_LOCATION_NEW, type: 'A', value: sysinfo.getIp() } ] })
-                    .reply(201, { ids: [ 'anotherdnsid' ] }, { 'Content-Type': 'application/json' })
-                    // app remove
-                    .delete('/api/v1/subdomains/anotherdnsid?token=' + config.token())
-                    .reply(204, { }, { 'Content-Type': 'application/json' });
+                    .replyWithFile(200, path.resolve(__dirname, '../../../webadmin/src/img/appicon_fallback.png'));
 
                 var port = parseInt(url.parse(config.apiServerOrigin()).port, 10);
-                hockServer = http.createServer(hockInstance.handler).listen(port, callback);
+                apiHockServer = http.createServer(apiHockInstance.handler).listen(port, callback);
+            },
+
+            settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
+
+            function (callback) {
+                awsHockInstance
+                    .get('/2013-04-01/hostedzone')
+                    .max(Infinity)
+                    .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }), { 'Content-Type': 'application/xml' })
+                    .filteringRequestBody(function (unusedBody) { return ''; }) // strip out body
+                    .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+                    .max(Infinity)
+                    .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'dnsrecordid', Status: 'INSYNC' } }), { 'Content-Type': 'application/xml' });
+
+                awsHockServer = http.createServer(awsHockInstance.handler).listen(5353, callback);
             }
         ], done);
     });
 
     after(function (done) {
         APP_ID = null;
-        cleanup(function (error) {
-            if (error) return done(error);
-            hockServer.close(function () {
-                dockerProxy.close(done);
-            });
-        });
+        async.series([
+            cleanup,
+            apiHockServer.close.bind(apiHockServer),
+            awsHockServer.close.bind(awsHockServer),
+            dockerProxy.close.bind(dockerProxy)
+        ], done);
     });
 
     var appResult = null, appEntry = null;
@@ -981,7 +1109,7 @@ describe('App installation - port bindings', function () {
 
         request.post(SERVER_URL + '/api/v1/apps/install')
               .query({ access_token: token })
-              .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: '' })
+              .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null, oauthProxy: false })
               .end(function (err, res) {
             expect(res.statusCode).to.equal(202);
             expect(fake.isDone()).to.be.ok();
@@ -1061,7 +1189,14 @@ describe('App installation - port bindings', function () {
     it('installation - running container has volume mounted', function (done) {
         docker.getContainer(appEntry.containerId).inspect(function (error, data) {
             expect(error).to.not.be.ok();
-            expect(data.Volumes['/app/data']).to.eql(paths.DATA_DIR + '/' + APP_ID + '/data');
+
+            // support newer docker versions
+            if (data.Volumes) {
+                expect(data.Volumes['/app/data']).to.eql(paths.DATA_DIR + '/' + APP_ID + '/data');
+            } else {
+                expect(data.Mounts.filter(function (mount) { return mount.Destination === '/app/data'; })[0].Source).to.eql(paths.DATA_DIR + '/' + APP_ID + '/data');
+            }
+
             done();
         });
     });
@@ -1099,10 +1234,7 @@ describe('App installation - port bindings', function () {
             expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
 
             function checkRedis() {
-                var isMac = os.platform() === 'darwin';
-                var client =
-                    isMac ? redis.createClient(parseInt(exportedRedisPort, 10), '127.0.0.1', { auth_pass: password })
-                          : redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
+                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
                 client.on('error', done);
                 client.set('key', 'value');
                 client.get('key', function (err, reply) {
@@ -1135,7 +1267,7 @@ describe('App installation - port bindings', function () {
     it('cannot reconfigure app with missing location', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
               .query({ access_token: token })
-              .send({ appId: APP_ID, password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: 'roleAdmin' })
+              .send({ appId: APP_ID, password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true })
               .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             done();
@@ -1145,7 +1277,57 @@ describe('App installation - port bindings', function () {
     it('cannot reconfigure app with missing accessRestriction', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
               .query({ access_token: token })
-              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, oauthProxy: false })
+              .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot reconfigure app with missing oauthProxy', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
+              .query({ access_token: token })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+              .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot reconfigure app with only the cert, no key', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
+              .query({ access_token: token })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true, cert: validCert1 })
+              .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot reconfigure app with only the key, no cert', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
+              .query({ access_token: token })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true, key: validKey1 })
+              .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot reconfigure app with cert not bein a string', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
+              .query({ access_token: token })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true, cert: 1234, key: validKey1 })
+              .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot reconfigure app with key not bein a string', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
+              .query({ access_token: token })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true, cert: validCert1, key: 1234 })
               .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
             done();
@@ -1155,7 +1337,7 @@ describe('App installation - port bindings', function () {
     it('non admin cannot reconfigure app', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
               .query({ access_token: token_1 })
-              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: 'roleAdmin' })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true })
               .end(function (err, res) {
             expect(res.statusCode).to.equal(403);
             done();
@@ -1165,7 +1347,7 @@ describe('App installation - port bindings', function () {
     it('can reconfigure app', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
               .query({ access_token: token })
-              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: 'roleAdmin' })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true })
               .end(function (err, res) {
             expect(res.statusCode).to.equal(202);
             checkConfigureStatus(0, done);
@@ -1224,10 +1406,7 @@ describe('App installation - port bindings', function () {
             expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
             expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
 
-            var isMac = os.platform() === 'darwin';
-            var client =
-                isMac ? redis.createClient(parseInt(exportedRedisPort, 10), '127.0.0.1', { auth_pass: password })
-                      : redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
+            var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
             client.on('error', done);
             client.set('key', 'value');
             client.get('key', function (err, reply) {
@@ -1239,6 +1418,26 @@ describe('App installation - port bindings', function () {
         });
     });
 
+    it('scheduler works after reconfiguration', function (done) {
+        async.retry({ times: 100, interval: 1000 }, function (callback) {
+            var data = safe.fs.readFileSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env', 'utf8');
+
+            if (data && data.indexOf('ECHO_SERVER_PORT=7172') !== -1) return callback();
+
+            callback(new Error('not run yet'));
+        }, done);
+    });
+
+    it('can reconfigure app with custom certificate', function (done) {
+        request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
+              .query({ access_token: token })
+              .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, oauthProxy: true, cert: validCert1, key: validKey1 })
+              .end(function (err, res) {
+            expect(res.statusCode).to.equal(202);
+            checkConfigureStatus(0, done);
+        });
+    });
+
     it('can stop app', function (done) {
         request.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/stop')
             .query({ access_token: token })
@@ -1302,9 +1501,13 @@ describe('App installation - port bindings', function () {
     });
 
     it('uninstalled - unregistered subdomain', function (done) {
-        hockInstance.done(function (error) { // checks if all the hockServer APIs were called
+        apiHockInstance.done(function (error) { // checks if all the apiHockServer APIs were called
             expect(!error).to.be.ok();
-            done();
+
+            awsHockInstance.done(function (error) {
+                expect(!error).to.be.ok();
+                done();
+            });
         });
     });
 

src/routes/test/backups-test.js

@@ -13,6 +13,7 @@ var appdb = require('../../appdb.js'),
     expect = require('expect.js'),
     request = require('superagent'),
     server = require('../../server.js'),
+    settings = require('../../settings.js'),
     nock = require('nock'),
     userdb = require('../../userdb.js');
 
@@ -51,7 +52,11 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, '' /* accessRestriction */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, false /* oauthProxy */, callback);
+        },
+
+        function createSettings(callback) {
+            settings.setBackupConfig({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' }, callback);
         }
     ], done);
 }
@@ -70,7 +75,7 @@ describe('Backups API', function () {
 
     describe('get', function () {
         it('cannot get backups with appstore request failing', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=APPSTORE_TOKEN').reply(401, {});
+            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(401, {});
 
             request.get(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })
@@ -82,7 +87,7 @@ describe('Backups API', function () {
         });
 
         it('can get backups', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=APPSTORE_TOKEN').reply(200, { backups: ['foo', 'bar']});
+            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(200, { backups: ['foo', 'bar']});
 
             request.get(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })
@@ -119,8 +124,8 @@ describe('Backups API', function () {
 
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
-                        .put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] })
-                        .reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             request.post(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })
@@ -141,4 +146,3 @@ describe('Backups API', function () {
         });
     });
 });
-

src/routes/test/clients-test.js

@@ -71,7 +71,7 @@ describe('OAuth Clients API', function () {
             it('fails', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: 'profile,roleUser' })
+                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(412);
@@ -87,7 +87,7 @@ describe('OAuth Clients API', function () {
 
             it('fails without token', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
-                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: 'profile,roleUser' })
+                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(401);
@@ -98,7 +98,7 @@ describe('OAuth Clients API', function () {
             it('fails without appId', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ redirectURI: 'http://foobar.com', scope: 'profile,roleUser' })
+                       .send({ redirectURI: 'http://foobar.com', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(400);
@@ -109,7 +109,7 @@ describe('OAuth Clients API', function () {
             it('fails with empty appId', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ appId: '', redirectURI: 'http://foobar.com', scope: 'profile,roleUser' })
+                       .send({ appId: '', redirectURI: 'http://foobar.com', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(400);
@@ -142,7 +142,7 @@ describe('OAuth Clients API', function () {
             it('fails without redirectURI', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ appId: 'someApp', scope: 'profile,roleUser' })
+                       .send({ appId: 'someApp', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(400);
@@ -153,7 +153,7 @@ describe('OAuth Clients API', function () {
             it('fails with empty redirectURI', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ appId: 'someApp', redirectURI: '', scope: 'profile,roleUser' })
+                       .send({ appId: 'someApp', redirectURI: '', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(400);
@@ -164,7 +164,7 @@ describe('OAuth Clients API', function () {
             it('fails with malformed redirectURI', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ appId: 'someApp', redirectURI: 'foobar', scope: 'profile,roleUser' })
+                       .send({ appId: 'someApp', redirectURI: 'foobar', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(400);
@@ -175,7 +175,7 @@ describe('OAuth Clients API', function () {
             it('succeeds', function (done) {
                 superagent.post(SERVER_URL + '/api/v1/oauth/clients')
                        .query({ access_token: token })
-                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: 'profile,roleUser' })
+                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com', scope: 'profile' })
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(201);
@@ -195,7 +195,7 @@ describe('OAuth Clients API', function () {
             id: '',
             appId: 'someAppId-0',
             redirectURI: 'http://some.callback0',
-            scope: 'profile,roleUser'
+            scope: 'profile'
         };
 
         before(function (done) {
@@ -297,183 +297,12 @@ describe('OAuth Clients API', function () {
         });
     });
 
-    describe('update', function () {
-        var CLIENT_0 = {
-            id: '',
-            appId: 'someAppId-0',
-            redirectURI: 'http://some.callback0',
-            scope: 'profile,roleUser'
-        };
-        var CLIENT_1 = {
-            id: '',
-            appId: 'someAppId-1',
-            redirectURI: 'http://some.callback1',
-            scope: 'profile,roleUser'
-        };
-
-        before(function (done) {
-            async.series([
-                server.start.bind(null),
-                database._clear.bind(null),
-
-                function (callback) {
-                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
-                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
-
-                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
-                           .query({ setupToken: 'somesetuptoken' })
-                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                           .end(function (error, result) {
-                        expect(error).to.not.be.ok();
-                        expect(result).to.be.ok();
-                        expect(result.statusCode).to.equal(201);
-                        expect(scope1.isDone()).to.be.ok();
-                        expect(scope2.isDone()).to.be.ok();
-
-                        // stash token for further use
-                        token = result.body.token;
-
-                        callback();
-                    });
-                },
-
-                settings.setDeveloperMode.bind(null, true),
-
-                function (callback) {
-                    superagent.post(SERVER_URL + '/api/v1/oauth/clients')
-                           .query({ access_token: token })
-                           .send({ appId: CLIENT_0.appId, redirectURI: CLIENT_0.redirectURI, scope: CLIENT_0.scope })
-                           .end(function (error, result) {
-                        expect(error).to.not.be.ok();
-                        expect(result.statusCode).to.equal(201);
-
-                        CLIENT_0 = result.body;
-
-                        callback();
-                    });
-                }
-            ], done);
-        });
-
-        after(cleanup);
-
-        describe('without developer mode', function () {
-            before(function (done) {
-                settings.setDeveloperMode(false, done);
-            });
-
-            it('fails', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ appId: CLIENT_1.appId, redirectURI: CLIENT_1.redirectURI })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(412);
-                    done();
-                });
-            });
-        });
-
-        describe('with developer mode', function () {
-            before(function (done) {
-                settings.setDeveloperMode(true, done);
-            });
-
-            it('fails without token', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .send({ appId: 'someApp', redirectURI: 'http://foobar.com' })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(401);
-                    done();
-                });
-            });
-
-
-            it('fails without appId', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ redirectURI: CLIENT_1.redirectURI })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-
-            it('fails with empty appId', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ appId: '', redirectURI: CLIENT_1.redirectURI })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-
-            it('fails without redirectURI', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ appId: CLIENT_1.appId })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-
-            it('fails with empty redirectURI', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ appId: CLIENT_1.appId, redirectURI: '' })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-
-            it('fails with malformed redirectURI', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ appId: CLIENT_1.appId, redirectURI: 'foobar' })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-
-            it('succeeds', function (done) {
-                superagent.put(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                       .query({ access_token: token })
-                       .send({ appId: CLIENT_1.appId, redirectURI: CLIENT_1.redirectURI })
-                       .end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(202);
-
-                    superagent.get(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_0.id)
-                           .query({ access_token: token })
-                           .end(function (error, result) {
-                        expect(error).to.be(null);
-                        expect(result.statusCode).to.equal(200);
-                        expect(result.body.appId).to.equal(CLIENT_1.appId);
-                        expect(result.body.redirectURI).to.equal(CLIENT_1.redirectURI);
-
-                        done();
-                   });
-                });
-            });
-        });
-    });
-
     describe('del', function () {
         var CLIENT_0 = {
             id: '',
             appId: 'someAppId-0',
             redirectURI: 'http://some.callback0',
-            scope: 'profile,roleUser'
+            scope: 'profile'
         };
 
         before(function (done) {
@@ -591,8 +420,8 @@ describe('Clients', function () {
         email: 'some@email.com',
         admin: true,
         salt: 'somesalt',
-        createdAt: (new Date()).toUTCString(),
-        modifiedAt: (new Date()).toUTCString(),
+        createdAt: (new Date()).toISOString(),
+        modifiedAt: (new Date()).toISOString(),
         resetToken: hat(256)
     };
 

src/routes/test/cloudron-test.js

@@ -10,11 +10,7 @@ var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
-    fs = require('fs'),
-    os = require('os'),
-    path = require('path'),
     nock = require('nock'),
-    paths = require('../../paths.js'),
     request = require('superagent'),
     server = require('../../server.js'),
     shell = require('../../shell.js');
@@ -28,6 +24,7 @@ var server;
 function setup(done) {
     nock.cleanAll();
     config.set('version', '0.5.0');
+    config.set('fqdn', 'localhost');
     server.start(done);
 }
 
@@ -167,101 +164,6 @@ describe('Cloudron', function () {
         });
     });
 
-    describe('Certificates API', function () {
-        var certFile, keyFile;
-
-        before(function (done) {
-            certFile = path.join(os.tmpdir(), 'host.cert');
-            fs.writeFileSync(certFile, 'test certificate');
-
-            keyFile = path.join(os.tmpdir(), 'host.key');
-            fs.writeFileSync(keyFile, 'test key');
-
-            async.series([
-                setup,
-
-                function (callback) {
-                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
-                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
-
-                    request.post(SERVER_URL + '/api/v1/cloudron/activate')
-                           .query({ setupToken: 'somesetuptoken' })
-                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
-                           .end(function (error, result) {
-                        expect(error).to.not.be.ok();
-                        expect(result).to.be.ok();
-                        expect(scope1.isDone()).to.be.ok();
-                        expect(scope2.isDone()).to.be.ok();
-
-                        // stash token for further use
-                        token = result.body.token;
-
-                        callback();
-                    });
-                },
-            ], done);
-        });
-
-        after(function (done) {
-            fs.unlinkSync(certFile);
-            fs.unlinkSync(keyFile);
-
-            cleanup(done);
-        });
-
-        it('cannot set certificate without token', function (done) {
-            request.post(SERVER_URL + '/api/v1/cloudron/certificate')
-                   .end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(401);
-                done();
-            });
-        });
-
-        it('cannot set certificate without certificate', function (done) {
-            request.post(SERVER_URL + '/api/v1/cloudron/certificate')
-                   .query({ access_token: token })
-                   .attach('key', keyFile, 'key')
-                   .end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('cannot set certificate without key', function (done) {
-            request.post(SERVER_URL + '/api/v1/cloudron/certificate')
-                   .query({ access_token: token })
-                   .attach('certificate', certFile, 'certificate')
-                   .end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(400);
-                done();
-            });
-        });
-
-        it('can set certificate', function (done) {
-            request.post(SERVER_URL + '/api/v1/cloudron/certificate')
-                   .query({ access_token: token })
-                   .attach('key', keyFile, 'key')
-                   .attach('certificate', certFile, 'certificate')
-                   .end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(202);
-                done();
-            });
-        });
-
-        it('did set the certificate', function (done) {
-            var cert = fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'));
-            expect(cert).to.eql(fs.readFileSync(certFile));
-
-            var key = fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'));
-            expect(key).to.eql(fs.readFileSync(keyFile));
-            done();
-        });
-    });
-
     describe('get config', function () {
         before(function (done) {
             async.series([
@@ -310,14 +212,15 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.apiServerOrigin).to.eql('http://localhost:6060');
                 expect(result.body.webServerOrigin).to.eql(null);
-                expect(result.body.fqdn).to.eql('localhost');
+                expect(result.body.fqdn).to.eql(config.fqdn());
                 expect(result.body.isCustomDomain).to.eql(false);
                 expect(result.body.progress).to.be.an('object');
                 expect(result.body.update).to.be.an('object');
-                expect(result.body.version).to.eql('0.5.0');
+                expect(result.body.version).to.eql(config.version());
                 expect(result.body.developerMode).to.be.a('boolean');
                 expect(result.body.size).to.eql(null);
                 expect(result.body.region).to.eql(null);
+                expect(result.body.memory).to.eql(0);
                 expect(result.body.cloudronName).to.be.a('string');
 
                 done();
@@ -325,7 +228,7 @@ describe('Cloudron', function () {
         });
 
         it('succeeds', function (done) {
-            var scope = nock(config.apiServerOrigin()).get('/api/v1/boxes/localhost?token=' + config.token()).reply(200, { box: { region: 'sfo', size: 'small' }});
+            var scope = nock(config.apiServerOrigin()).get('/api/v1/boxes/localhost?token=' + config.token()).reply(200, { box: { region: 'sfo', size: '1gb' }});
 
             request.get(SERVER_URL + '/api/v1/cloudron/config')
                    .query({ access_token: token })
@@ -334,14 +237,15 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(200);
                 expect(result.body.apiServerOrigin).to.eql('http://localhost:6060');
                 expect(result.body.webServerOrigin).to.eql(null);
-                expect(result.body.fqdn).to.eql('localhost');
+                expect(result.body.fqdn).to.eql(config.fqdn());
                 expect(result.body.isCustomDomain).to.eql(false);
                 expect(result.body.progress).to.be.an('object');
                 expect(result.body.update).to.be.an('object');
-                expect(result.body.version).to.eql('0.5.0');
+                expect(result.body.version).to.eql(config.version());
                 expect(result.body.developerMode).to.be.a('boolean');
-                expect(result.body.size).to.eql('small');
+                expect(result.body.size).to.eql('1gb');
                 expect(result.body.region).to.eql('sfo');
+                expect(result.body.memory).to.eql(1073741824);
                 expect(result.body.cloudronName).to.be.a('string');
 
                 expect(scope.isDone()).to.be.ok();
@@ -378,6 +282,19 @@ describe('Cloudron', function () {
                         callback();
                     });
                 },
+
+                function setupBackupConfig(callback) {
+                    request.post(SERVER_URL + '/api/v1/settings/backup_config')
+                           .send({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(error).to.not.be.ok();
+                        expect(result.statusCode).to.equal(200);
+
+                        callback();
+                    });
+                }
+
             ], done);
         });
 
@@ -437,7 +354,6 @@ describe('Cloudron', function () {
             });
         });
 
-
         it('fails with wrong region type', function (done) {
             request.post(SERVER_URL + '/api/v1/cloudron/migrate')
                    .send({ size: 'small', region: 4, password: PASSWORD })
@@ -450,8 +366,20 @@ describe('Cloudron', function () {
         });
 
         it('fails when in wrong state', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(409, {});
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                    return body.size && body.region && body.restoreKey;
+                }).reply(409, {});
 
             injectShellMock();
 
@@ -463,7 +391,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }
@@ -475,10 +403,20 @@ describe('Cloudron', function () {
             });
         });
 
-
         it('succeeds', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(202, {});
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                return body.size && body.region && body.restoreKey;
+            }).reply(202, {});
+
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) {
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             injectShellMock();
 
@@ -490,7 +428,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }

src/routes/test/settings-test.js

@@ -11,6 +11,7 @@ var appdb = require('../../appdb.js'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
+    path = require('path'),
     paths = require('../../paths.js'),
     request = require('superagent'),
     server = require('../../server.js'),
@@ -26,6 +27,8 @@ var token = null;
 
 var server;
 function setup(done) {
+    config.set('fqdn', 'foobar.com');
+
     async.series([
         server.start.bind(server),
 
@@ -54,7 +57,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, '' /* accessRestriction */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, false /* oauthProxy */, callback);
         }
     ], done);
 }
@@ -212,7 +215,7 @@ describe('Settings API', function () {
         it('set succeeds', function (done) {
             request.post(SERVER_URL + '/api/v1/settings/cloudron_avatar')
                    .query({ access_token: token })
-                   .attach('avatar', paths.FAVICON_FILE)
+                   .attach('avatar', paths.CLOUDRON_DEFAULT_AVATAR_FILE)
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 done();
@@ -224,10 +227,146 @@ describe('Settings API', function () {
                    .query({ access_token: token })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(200);
-                expect(res.body.toString()).to.eql(fs.readFileSync(paths.FAVICON_FILE, 'utf-8'));
+                expect(res.body.toString()).to.eql(fs.readFileSync(paths.CLOUDRON_DEFAULT_AVATAR_FILE, 'utf-8'));
                 done(err);
             });
         });
     });
+
+    describe('dns_config', function () {
+        it('get dns_config fails', function (done) {
+            request.get(SERVER_URL + '/api/v1/settings/dns_config')
+                   .query({ access_token: token })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body).to.eql({});
+                done(err);
+            });
+        });
+
+        it('cannot set without data', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/dns_config')
+                   .query({ access_token: token })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('set succeeds', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/dns_config')
+                   .query({ access_token: token })
+                   .send({ provider: 'route53', accessKeyId: 'accessKey', secretAccessKey: 'secretAccessKey' })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                done();
+            });
+        });
+
+        it('get succeeds', function (done) {
+            request.get(SERVER_URL + '/api/v1/settings/dns_config')
+                   .query({ access_token: token })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(200);
+                expect(res.body).to.eql({ provider: 'route53', accessKeyId: 'accessKey', secretAccessKey: 'secretAccessKey', region: 'us-east-1', endpoint: null });
+                done(err);
+            });
+        });
+    });
+
+    describe('Certificates API', function () {
+        // foobar.com
+        var validCert0 = '-----BEGIN CERTIFICATE-----\nMIIBujCCAWQCCQCjLyTKzAJ4FDANBgkqhkiG9w0BAQsFADBkMQswCQYDVQQGEwJE\nRTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xEDAOBgNVBAoMB05l\nYnVsb24xDDAKBgNVBAsMA0NUTzETMBEGA1UEAwwKZm9vYmFyLmNvbTAeFw0xNTEw\nMjgxMjM5MjZaFw0xNjEwMjcxMjM5MjZaMGQxCzAJBgNVBAYTAkRFMQ8wDQYDVQQI\nDAZCZXJsaW4xDzANBgNVBAcMBkJlcmxpbjEQMA4GA1UECgwHTmVidWxvbjEMMAoG\nA1UECwwDQ1RPMRMwEQYDVQQDDApmb29iYXIuY29tMFwwDQYJKoZIhvcNAQEBBQAD\nSwAwSAJBAMeYofgwHeNVmGkGe0gj4dnX2ciifDi7X2K/oVHp7mxuHjGMSYP9Z7b6\n+mu0IMf4OedwXStHBeO8mwjKxZmE7p8CAwEAATANBgkqhkiG9w0BAQsFAANBAJI7\nFUUHXjR63UFk8pgxp0c7hEGqj4VWWGsmo8oZnnX8jGVmQDKbk8o3MtDujfqupmMR\nMo7tSAFlG7zkm3GYhpw=\n-----END CERTIFICATE-----';
+        var validKey0 = '-----BEGIN RSA PRIVATE KEY-----\nMIIBOwIBAAJBAMeYofgwHeNVmGkGe0gj4dnX2ciifDi7X2K/oVHp7mxuHjGMSYP9\nZ7b6+mu0IMf4OedwXStHBeO8mwjKxZmE7p8CAwEAAQJBAJS59Sb8o6i8JT9NJxvQ\nMQCkSJGqEaosZJ0uccSZ7aE48v+H7HiPzXAueitohcEif2Wp1EZ1RbRMURhznNiZ\neLECIQDxxqhakO6wc7H68zmpRXJ5ZxGUNbM24AMtpONAtEw9iwIhANNWtp6P74OV\ntvfOmtubbqw768fmGskFCOcp5oF8oF29AiBkTAf9AhCyjFwyAYJTEScq67HkLN66\njfVjkvpfFixmfwIgI+xldmZ5DCDyzQSthg7RrS0yUvRmMS1N6h1RNUl96PECIQDl\nit4lFcytbqNo1PuBZvzQE+plCjiJqXHYo3WCst1Jbg==\n-----END RSA PRIVATE KEY-----';
+
+        // *.foobar.com
+        var validCert1 = '-----BEGIN CERTIFICATE-----\nMIIBvjCCAWgCCQCg957GWuHtbzANBgkqhkiG9w0BAQsFADBmMQswCQYDVQQGEwJE\nRTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4xEDAOBgNVBAoMB05l\nYnVsb24xDDAKBgNVBAsMA0NUTzEVMBMGA1UEAwwMKi5mb29iYXIuY29tMB4XDTE1\nMTAyODEzMDI1MFoXDTE2MTAyNzEzMDI1MFowZjELMAkGA1UEBhMCREUxDzANBgNV\nBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMRAwDgYDVQQKDAdOZWJ1bG9uMQww\nCgYDVQQLDANDVE8xFTATBgNVBAMMDCouZm9vYmFyLmNvbTBcMA0GCSqGSIb3DQEB\nAQUAA0sAMEgCQQC0FKf07ZWMcABFlZw+GzXK9EiZrlJ1lpnu64RhN99z7MXRr8cF\nnZVgY3jgatuyR5s3WdzUvye2eJ0rNicl2EZJAgMBAAEwDQYJKoZIhvcNAQELBQAD\nQQAw4bteMZAeJWl2wgNLw+wTwAH96E0jyxwreCnT5AxJLmgimyQ0XOF4FsssdRFj\nxD9WA+rktelBodJyPeTDNhIh\n-----END CERTIFICATE-----';
+        var validKey1 = '-----BEGIN RSA PRIVATE KEY-----\nMIIBOQIBAAJBALQUp/TtlYxwAEWVnD4bNcr0SJmuUnWWme7rhGE333PsxdGvxwWd\nlWBjeOBq27JHmzdZ3NS/J7Z4nSs2JyXYRkkCAwEAAQJALV2eykcoC48TonQEPmkg\nbhaIS57syw67jMLsQImQ02UABKzqHPEKLXPOZhZPS9hsC/hGIehwiYCXMUlrl+WF\nAQIhAOntBI6qaecNjAAVG7UbZclMuHROUONmZUF1KNq6VyV5AiEAxRLkfHWy52CM\njOQrX347edZ30f4QczvugXwsyuU9A1ECIGlGZ8Sk4OBA8n6fAUcyO06qnmCJVlHg\npTUeOvKk5c9RAiBs28+8dCNbrbhVhx/yQr9FwNM0+ttJW/yWJ+pyNQhr0QIgJTT6\nxwCWYOtbioyt7B9l+ENy3AMSO3Uq+xmIKkvItK4=\n-----END RSA PRIVATE KEY-----';
+
+        it('cannot set certificate without token', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('cannot set certificate without certificate', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .query({ access_token: token })
+                   .send({ key: validKey1 })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot set certificate without key', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .query({ access_token: token })
+                   .send({ cert: validCert1 })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot set certificate with cert not being a string', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .query({ access_token: token })
+                   .send({ cert: 1234, key: validKey1 })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot set certificate with key not being a string', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .query({ access_token: token })
+                   .send({ cert: validCert1, key: true })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot set non wildcard certificate', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .query({ access_token: token })
+                   .send({ cert: validCert0, key: validKey0 })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('can set certificate', function (done) {
+            request.post(SERVER_URL + '/api/v1/settings/certificate')
+                   .query({ access_token: token })
+                   .send({ cert: validCert1, key: validKey1 })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+
+        it('did set the certificate', function (done) {
+            var cert = fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf-8');
+            expect(cert).to.eql(validCert1);
+
+            var key = fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf-8');
+            expect(key).to.eql(validKey1);
+
+            done();
+        });
+    });
 });
 

src/routes/test/simpleauth-test.js

@@ -0,0 +1,421 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var clientdb = require('../../clientdb.js'),
+    appdb = require('../../appdb.js'),
+    async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    request = require('superagent'),
+    server = require('../../server.js'),
+    simpleauth = require('../../simpleauth.js'),
+    nock = require('nock');
+
+describe('SimpleAuth API', function () {
+    var SERVER_URL = 'http://localhost:' + config.get('port');
+    var SIMPLE_AUTH_ORIGIN = 'http://localhost:' + config.get('simpleAuthPort');
+
+    var USERNAME = 'admin', PASSWORD = 'password', EMAIL ='silly@me.com';
+
+    var APP_0 = {
+        id: 'app0',
+        appStoreId: '',
+        manifest: { version: '0.1.0', addons: { } },
+        location: 'test0',
+        portBindings: {},
+        accessRestriction: { users: [ 'foobar', 'someone'] },
+        oauthProxy: true
+    };
+
+    var APP_1 = {
+        id: 'app1',
+        appStoreId: '',
+        manifest: { version: '0.1.0', addons: { } },
+        location: 'test1',
+        portBindings: {},
+        accessRestriction: { users: [ 'foobar', USERNAME, 'someone' ] },
+        oauthProxy: true
+    };
+
+    var APP_2 = {
+        id: 'app2',
+        appStoreId: '',
+        manifest: { version: '0.1.0', addons: { } },
+        location: 'test2',
+        portBindings: {},
+        accessRestriction: null,
+        oauthProxy: true
+    };
+
+    var CLIENT_0 = {
+        id: 'someclientid',
+        appId: 'someappid',
+        type: clientdb.TYPE_SIMPLE_AUTH,
+        clientSecret: 'someclientsecret',
+        redirectURI: '',
+        scope: 'user,profile'
+    };
+
+    var CLIENT_1 = {
+        id: 'someclientid1',
+        appId: APP_0.id,
+        type: clientdb.TYPE_SIMPLE_AUTH,
+        clientSecret: 'someclientsecret1',
+        redirectURI: '',
+        scope: 'user,profile'
+    };
+
+    var CLIENT_2 = {
+        id: 'someclientid2',
+        appId: APP_1.id,
+        type: clientdb.TYPE_SIMPLE_AUTH,
+        clientSecret: 'someclientsecret2',
+        redirectURI: '',
+        scope: 'user,profile'
+    };
+
+    var CLIENT_3 = {
+        id: 'someclientid3',
+        appId: APP_2.id,
+        type: clientdb.TYPE_SIMPLE_AUTH,
+        clientSecret: 'someclientsecret3',
+        redirectURI: '',
+        scope: 'user,profile'
+    };
+
+    var CLIENT_4 = {
+        id: 'someclientid4',
+        appId: APP_2.id,
+        type: clientdb.TYPE_OAUTH,
+        clientSecret: 'someclientsecret4',
+        redirectURI: '',
+        scope: 'user,profile'
+    };
+
+    before(function (done) {
+        async.series([
+            server.start.bind(server),
+            simpleauth.start.bind(simpleauth),
+
+            database._clear,
+
+            function createAdmin(callback) {
+                var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                request.post(SERVER_URL + '/api/v1/cloudron/activate')
+                       .query({ setupToken: 'somesetuptoken' })
+                       .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                       .end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result).to.be.ok();
+                    expect(result.statusCode).to.eql(201);
+                    expect(scope1.isDone()).to.be.ok();
+                    expect(scope2.isDone()).to.be.ok();
+
+                    callback();
+                });
+            },
+
+            clientdb.add.bind(null, CLIENT_0.id, CLIENT_0.appId, CLIENT_0.type, CLIENT_0.clientSecret, CLIENT_0.redirectURI, CLIENT_0.scope),
+            clientdb.add.bind(null, CLIENT_1.id, CLIENT_1.appId, CLIENT_1.type, CLIENT_1.clientSecret, CLIENT_1.redirectURI, CLIENT_1.scope),
+            clientdb.add.bind(null, CLIENT_2.id, CLIENT_2.appId, CLIENT_2.type, CLIENT_2.clientSecret, CLIENT_2.redirectURI, CLIENT_2.scope),
+            clientdb.add.bind(null, CLIENT_3.id, CLIENT_3.appId, CLIENT_3.type, CLIENT_3.clientSecret, CLIENT_3.redirectURI, CLIENT_3.scope),
+            clientdb.add.bind(null, CLIENT_4.id, CLIENT_4.appId, CLIENT_4.type, CLIENT_4.clientSecret, CLIENT_4.redirectURI, CLIENT_4.scope),
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.oauthProxy),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.oauthProxy),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.oauthProxy)
+        ], done);
+    });
+
+    after(function (done) {
+        async.series([
+            database._clear,
+            simpleauth.stop.bind(simpleauth),
+            server.stop.bind(server)
+        ], done);
+    });
+
+    describe('login', function () {
+        it('cannot login without clientId', function (done) {
+            var body = {};
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot login without username', function (done) {
+            var body = {
+                clientId: 'someclientid'
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot login without password', function (done) {
+            var body = {
+                clientId: 'someclientid',
+                username: USERNAME
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('cannot login with unkown clientId', function (done) {
+            var body = {
+                clientId: CLIENT_0.id+CLIENT_0.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('cannot login with unkown user', function (done) {
+            var body = {
+                clientId: CLIENT_0.id,
+                username: USERNAME+USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('cannot login with empty password', function (done) {
+            var body = {
+                clientId: CLIENT_0.id,
+                username: USERNAME,
+                password: ''
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('cannot login with wrong password', function (done) {
+            var body = {
+                clientId: CLIENT_0.id,
+                username: USERNAME,
+                password: PASSWORD+PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails for unkown app', function (done) {
+            var body = {
+                clientId: CLIENT_0.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails for disallowed app', function (done) {
+            var body = {
+                clientId: CLIENT_1.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds for allowed app', function (done) {
+            var body = {
+                clientId: CLIENT_2.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.accessToken).to.be.a('string');
+                expect(result.body.user).to.be.an('object');
+                expect(result.body.user.id).to.be.a('string');
+                expect(result.body.user.username).to.be.a('string');
+                expect(result.body.user.email).to.be.a('string');
+                expect(result.body.user.admin).to.be.a('boolean');
+
+                request.get(SERVER_URL + '/api/v1/profile')
+                .query({ access_token: result.body.accessToken })
+                .end(function (error, result) {
+                    expect(error).to.be(null);
+                    expect(result.body).to.be.an('object');
+                    expect(result.body.username).to.eql(USERNAME);
+
+                    done();
+                });
+            });
+        });
+
+        it('succeeds for app without accessRestriction', function (done) {
+            var body = {
+                clientId: CLIENT_3.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.accessToken).to.be.a('string');
+                expect(result.body.user).to.be.an('object');
+                expect(result.body.user.id).to.be.a('string');
+                expect(result.body.user.username).to.be.a('string');
+                expect(result.body.user.email).to.be.a('string');
+                expect(result.body.user.admin).to.be.a('boolean');
+
+                request.get(SERVER_URL + '/api/v1/profile')
+                .query({ access_token: result.body.accessToken })
+                .end(function (error, result) {
+                    expect(error).to.be(null);
+                    expect(result.body).to.be.an('object');
+                    expect(result.body.username).to.eql(USERNAME);
+
+                    done();
+                });
+            });
+        });
+
+        it('fails for wrong client credentials', function (done) {
+            var body = {
+                clientId: CLIENT_4.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+    });
+
+    describe('logout', function () {
+        var accessToken;
+
+        before(function (done) {
+            var body = {
+                clientId: CLIENT_3.id,
+                username: USERNAME,
+                password: PASSWORD
+            };
+
+            request.post(SIMPLE_AUTH_ORIGIN + '/api/v1/login')
+            .send(body)
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+
+                accessToken = result.body.accessToken;
+
+                done();
+            });
+        });
+
+        it('fails without access_token', function (done) {
+            request.get(SIMPLE_AUTH_ORIGIN + '/api/v1/logout')
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails with unkonwn access_token', function (done) {
+            request.get(SIMPLE_AUTH_ORIGIN + '/api/v1/logout')
+            .query({ access_token: accessToken+accessToken })
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            request.get(SIMPLE_AUTH_ORIGIN + '/api/v1/logout')
+            .query({ access_token: accessToken })
+            .end(function (error, result) {
+                expect(error).to.be(null);
+                expect(result.statusCode).to.equal(200);
+
+                request.get(SERVER_URL + '/api/v1/profile')
+                .query({ access_token: accessToken })
+                .end(function (error, result) {
+                    expect(error).to.be(null);
+                    expect(result.statusCode).to.equal(401);
+
+                    done();
+                });
+            });
+        });
+    });
+});

src/routes/test/start_addons.sh

@@ -14,7 +14,7 @@ root_password=secret
 start_postgresql() {
     postgresql_vars="POSTGRESQL_ROOT_PASSWORD=${root_password}; POSTGRESQL_ROOT_HOST=172.17.0.0/255.255.0.0"
 
-    if which boot2docker >/dev/null; then
+    if which boot2docker >/dev/null 2>&1; then
         boot2docker ssh "sudo rm -rf /tmp/postgresql_vars.sh"
         boot2docker ssh "echo \"${postgresql_vars}\" > /tmp/postgresql_vars.sh"
     else
@@ -24,13 +24,15 @@ start_postgresql() {
 
     docker rm -f postgresql 2>/dev/null 1>&2 || true
 
-    docker run -dtP --name=postgresql -v "${postgresqldatadir}:/var/lib/postgresql" -v /tmp/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh "${POSTGRESQL_IMAGE}" >/dev/null
+    docker run -dtP --name=postgresql -v "${postgresqldatadir}:/var/lib/postgresql" \
+        --read-only -v /tmp -v /run \
+        -v /tmp/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh "${POSTGRESQL_IMAGE}" >/dev/null
 }
 
 start_mysql() {
     local mysql_vars="MYSQL_ROOT_PASSWORD=${root_password}; MYSQL_ROOT_HOST=172.17.0.0/255.255.0.0"
 
-    if which boot2docker >/dev/null; then
+    if which boot2docker >/dev/null 2>&1; then
         boot2docker ssh "sudo rm -rf /tmp/mysql_vars.sh"
         boot2docker ssh "echo \"${mysql_vars}\" > /tmp/mysql_vars.sh"
     else
@@ -40,13 +42,15 @@ start_mysql() {
 
     docker rm -f mysql 2>/dev/null 1>&2 || true
 
-    docker run -dP --name=mysql -v "${mysqldatadir}:/var/lib/mysql" -v /tmp/mysql_vars.sh:/etc/mysql/mysql_vars.sh "${MYSQL_IMAGE}" >/dev/null
+    docker run -dP --name=mysql -v "${mysqldatadir}:/var/lib/mysql" \
+        --read-only -v /tmp -v /run \
+        -v /tmp/mysql_vars.sh:/etc/mysql/mysql_vars.sh "${MYSQL_IMAGE}" >/dev/null
 }
 
 start_mongodb() {
     local mongodb_vars="MONGODB_ROOT_PASSWORD=${root_password}"
 
-    if which boot2docker >/dev/null; then
+    if which boot2docker >/dev/null 2>&1; then
         boot2docker ssh "sudo rm -rf /tmp/mongodb_vars.sh"
         boot2docker ssh "echo \"${mongodb_vars}\" > /tmp/mongodb_vars.sh"
     else
@@ -56,16 +60,27 @@ start_mongodb() {
 
     docker rm -f mongodb 2>/dev/null 1>&2 || true
 
-    docker run -dP --name=mongodb -v "${mongodbdatadir}:/var/lib/mongodb" -v /tmp/mongodb_vars.sh:/etc/mongodb_vars.sh "${MONGODB_IMAGE}" >/dev/null
+    docker run -dP --name=mongodb -v "${mongodbdatadir}:/var/lib/mongodb" \
+        --read-only -v /tmp -v /run \
+        -v /tmp/mongodb_vars.sh:/etc/mongodb_vars.sh "${MONGODB_IMAGE}" >/dev/null
+}
+
+start_mail() {
+    docker rm -f mail 2>/dev/null 1>&2 || true
+
+    docker run -dP --name=mail -e DOMAIN_NAME="localhost" \
+        --read-only -v /tmp -v /run \
+        -v /tmp/maildata:/app/data "${MAIL_IMAGE}" >/dev/null
 }
 
 start_mysql
 start_postgresql
 start_mongodb
+start_mail
 
 echo -n "Waiting for addons to start"
-for i in {1..10}; do
-   echo -n "."
+for i in {1..20}; do
+    echo -n "."
     sleep 1
 done
 echo ""

src/scheduler.js

@@ -0,0 +1,195 @@
+'use strict';
+
+exports = module.exports = {
+    sync: sync
+};
+
+var appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    CronJob = require('cron').CronJob,
+    debug = require('debug')('box:src/scheduler'),
+    docker = require('./docker.js'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+var NOOP_CALLBACK = function (error) { if (error) debug('Unhandled error: ', error); };
+
+// appId -> { schedulerConfig (manifest), cronjobs, containerIds }
+var gState = (function loadState() {
+    var state = safe.JSON.parse(safe.fs.readFileSync(paths.SCHEDULER_FILE, 'utf8'));
+    return state || { };
+})();
+
+function saveState(state) {
+    // do not save cronJobs
+    var safeState = { };
+    for (var appId in state) {
+        safeState[appId] = {
+            schedulerConfig: state[appId].schedulerConfig,
+            containerIds: state[appId].containerIds
+        };
+    }
+    safe.fs.writeFileSync(paths.SCHEDULER_FILE, JSON.stringify(safeState, null, 4), 'utf8');
+}
+
+function sync(callback) {
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Syncing');
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(error);
+
+        // stop tasks of apps that went away
+        var allAppIds = allApps.map(function (app) { return app.id; });
+        var removedAppIds = _.difference(Object.keys(gState), allAppIds);
+        async.eachSeries(removedAppIds, function (appId, iteratorDone) {
+            stopJobs(appId, gState[appId], true /* killContainers */, iteratorDone);
+        }, function (error) {
+            if (error) debug('Error stopping jobs : %j', error);
+
+            gState = _.omit(gState, removedAppIds);
+
+            // start tasks of new apps
+            async.eachSeries(allApps, function (app, iteratorDone) {
+                var appState = gState[app.id] || null;
+                var schedulerConfig = app.manifest.addons.scheduler || null;
+
+                if (!appState && !schedulerConfig) return iteratorDone(); // nothing changed
+
+                if (appState && _.isEqual(appState.schedulerConfig, schedulerConfig) && appState.cronJobs) {
+                    return iteratorDone(); // nothing changed
+                }
+
+                var killContainers = appState && !appState.cronJobs ? true : false; // keep the old containers on 'startup'
+                stopJobs(app.id, appState, killContainers, function (error) {
+                    if (error) debug('Error stopping jobs for %s : %s', app.id, error.message);
+
+                    if (!schedulerConfig) {
+                        delete gState[app.id];
+                        return iteratorDone();
+                    }
+
+                    gState[app.id] = {
+                        schedulerConfig: schedulerConfig,
+                        cronJobs: createCronJobs(app.id, schedulerConfig),
+                        containerIds: { }
+                    };
+
+                    saveState(gState);
+
+                    iteratorDone();
+                });
+            });
+
+            debug('Done syncing');
+        });
+    });
+}
+
+function killContainer(containerId, callback) {
+    if (!containerId) return callback();
+
+    async.series([
+        docker.stopContainer.bind(null, containerId),
+        docker.deleteContainer.bind(null, containerId)
+    ], function (error) {
+        if (error) debug('Failed to kill task with containerId %s : %s', containerId, error.message);
+
+        callback(error);
+    });
+}
+
+function stopJobs(appId, appState, killContainers, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof appState, 'object');
+    assert.strictEqual(typeof killContainers, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('stopJobs for %s', appId);
+
+    if (!appState) return callback();
+
+    async.eachSeries(Object.keys(appState.schedulerConfig), function (taskName, iteratorDone) {
+        if (appState.cronJobs && appState.cronJobs[taskName]) {  // could be null across restarts
+            appState.cronJobs[taskName].stop();
+        }
+
+        if (!killContainers) return iteratorDone();
+
+        killContainer(appState.containerIds[taskName], iteratorDone);
+    }, callback);
+}
+
+function createCronJobs(appId, schedulerConfig) {
+    assert.strictEqual(typeof appId, 'string');
+    assert(schedulerConfig && typeof schedulerConfig === 'object');
+
+    debug('creating cron jobs for app %s', appId);
+
+    var jobs = { };
+
+    Object.keys(schedulerConfig).forEach(function (taskName) {
+        var task = schedulerConfig[taskName];
+
+        var cronTime = (config.TEST ? '*/5 ' : '00 ') + task.schedule; // time ticks faster in tests
+
+        debug('scheduling task for %s/%s @ %s : %s', appId, taskName, cronTime, task.command);
+
+        var cronJob = new CronJob({
+            cronTime: cronTime, // at this point, the pattern has been validated
+            onTick: doTask.bind(null, appId, taskName),
+            start: true
+        });
+
+        jobs[taskName] = cronJob;
+    });
+
+    return jobs;
+}
+
+function doTask(appId, taskName, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof taskName, 'string');
+    assert(!callback || typeof callback === 'function');
+
+    callback = callback || NOOP_CALLBACK;
+
+    var appState = gState[appId];
+
+    debug('Executing task %s/%s', appId, taskName);
+
+    apps.get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
+            debug('task %s skipped. app %s is not installed/running', taskName, app.id);
+            return callback();
+        }
+
+        if (appState.containerIds[taskName]) debug('task %s/%s has existing container %s. killing it', appId, taskName, appState.containerIds[taskName]);
+
+        killContainer(appState.containerIds[taskName], function (error) {
+            if (error) return callback(error);
+
+            debug('Creating createSubcontainer for %s/%s : %s', app.id, taskName, gState[appId].schedulerConfig[taskName].command);
+
+            // NOTE: if you change container name here, fix addons.js to return correct container names
+            docker.createSubcontainer(app, app.id + '-' + taskName, [ '/bin/sh', '-c', gState[appId].schedulerConfig[taskName].command ], { } /* options */, function (error, container) {
+                if (error) return callback(error);
+
+                appState.containerIds[taskName] = container.id;
+
+                saveState(gState);
+
+                docker.startContainer(container.id, callback);
+            });
+        });
+    });
+}

src/scripts/backupapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: backup.sh <appid> <url> <key>"
+    echo "Usage: backupapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -22,6 +22,7 @@ readonly DATA_DIR="${HOME}/data"
 app_id="$1"
 backup_url="$2"
 backup_key="$3"
+session_token="$4"
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
@@ -31,9 +32,17 @@ btrfs subvolume snapshot -r "${app_data_dir}" "${app_data_snapshot}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${app_data_snapshot}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT "${headers[@]}" --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/backupbox.sh

@@ -13,12 +13,13 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 2 ]; then
-    echo "Usage: backupbox.sh <url> <key>"
+    echo "Usage: backupbox.sh <url> <key> [aws session token]"
     exit 1
 fi
 
 backup_url="$1"
 backup_key="$2"
+session_token="$3"
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"
@@ -32,9 +33,17 @@ btrfs subvolume snapshot -r "${BOX_DATA_DIR}" "${box_snapshot_dir}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${box_snapshot_dir}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/collectlogs.sh

@@ -21,7 +21,7 @@ readonly program_name=$1
 
 echo "${program_name}.log"
 echo "-------------------"
-tail --lines=100 /var/log/supervisor/${program_name}.log
+journalctl --no-pager -u ${program_name} -n 100
 echo
 echo
 echo "dmesg"
@@ -31,7 +31,7 @@ echo
 echo
 echo "docker"
 echo "------"
-tail --lines=100 /var/log/upstart/docker.log
+journalctl --no-pager -u docker -n 50
 echo
 echo
 

src/scripts/reloadnginx.sh

@@ -12,12 +12,6 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${OSTYPE}" == "darwin"* ]]; then
-    # On Mac, brew installs supervisor in /usr/local/bin
-    export PATH=$PATH:/usr/local/bin
-fi
-
 if [[ "${BOX_ENV}" == "cloudron" ]]; then
     nginx -s reload
 fi
-

src/scripts/restoreapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: restoreapp.sh <appid> <url> <key>"
+    echo "Usage: restoreapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -23,6 +23,7 @@ readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max
 app_id="$1"
 restore_url="$2"
 restore_key="$3"
+session_token="$4"
 
 echo "Downloading backup: ${restore_url} and key: ${restore_key}"
 
@@ -30,7 +31,14 @@ for try in `seq 1 5`; do
     echo "Download backup from ${restore_url} (try ${try})"
     error_log=$(mktemp)
 
-    if $curl -L "${restore_url}" \
+    headers=("") # empty element required (http://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u)
+
+    # federated tokens in CaaS case need session token
+    if [[ ! -z "${session_token}" ]]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
+    if $curl -L "${headers[@]}" "${restore_url}" \
         | openssl aes-256-cbc -d -pass "pass:${restore_key}" \
         | tar -zxf - -C "${DATA_DIR}/${app_id}" 2>"${error_log}"; then
         chown -R yellowtent:yellowtent "${DATA_DIR}/${app_id}"

src/server.js

@@ -20,7 +20,6 @@ var assert = require('assert'),
     middleware = require('./middleware'),
     passport = require('passport'),
     path = require('path'),
-    paths = require('./paths.js'),
     routes = require('./routes/index.js'),
     taskmanager = require('./taskmanager.js');
 
@@ -43,11 +42,7 @@ function initializeExpressSync() {
     app.set('view options', { layout: true, debug: true });
     app.set('view engine', 'ejs');
 
-    if (process.env.BOX_ENV === 'test') {
-       app.use(express.static(path.join(__dirname, '/../webadmin')));
-    } else {
-        app.use(middleware.morgan('dev', { immediate: false }));
-    }
+    if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('Box :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
 
     var router = new express.Router();
     router.del = router.delete; // amend router.del for readability further on
@@ -57,7 +52,6 @@ function initializeExpressSync() {
        .use(json)
        .use(urlencoded)
        .use(middleware.cookieParser())
-       .use(middleware.favicon(paths.FAVICON_FILE)) // used when serving oauth login page
        .use(middleware.cors({ origins: [ '*' ], allowCredentials: true }))
        .use(middleware.session({ secret: 'yellow is blue', resave: true, saveUninitialized: true, cookie: { path: '/', httpOnly: true, secure: false, maxAge: 600000 } }))
        .use(passport.initialize())
@@ -97,9 +91,8 @@ function initializeExpressSync() {
     // private routes
     router.get ('/api/v1/cloudron/config', rootScope, routes.cloudron.getConfig);
     router.post('/api/v1/cloudron/update', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
-    router.get ('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
+    router.post('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
     router.post('/api/v1/cloudron/migrate', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.migrate);
-    router.post('/api/v1/cloudron/certificate', rootScope, multipart, routes.cloudron.setCertificate);
     router.get ('/api/v1/cloudron/graphs', rootScope, routes.graphs.getGraphs);
 
     // feedback
@@ -120,7 +113,6 @@ function initializeExpressSync() {
     router.post('/api/v1/session/login', csrf, routes.oauth2.login);
     router.get ('/api/v1/session/logout', routes.oauth2.logout);
     router.get ('/api/v1/session/callback', routes.oauth2.callback);
-    router.get ('/api/v1/session/error', routes.oauth2.error);
     router.get ('/api/v1/session/password/resetRequest.html', csrf, routes.oauth2.passwordResetRequestSite);
     router.post('/api/v1/session/password/resetRequest', csrf, routes.oauth2.passwordResetRequest);
     router.get ('/api/v1/session/password/sent.html', routes.oauth2.passwordSentSite);
@@ -130,13 +122,11 @@ function initializeExpressSync() {
 
     // oauth2 routes
     router.get ('/api/v1/oauth/dialog/authorize', routes.oauth2.authorization);
-    router.post('/api/v1/oauth/dialog/authorize/decision', csrf, routes.oauth2.decision);
     router.post('/api/v1/oauth/token', routes.oauth2.token);
     router.get ('/api/v1/oauth/clients', settingsScope, routes.clients.getAllByUserId);
     router.post('/api/v1/oauth/clients', routes.developer.enabled, settingsScope, routes.clients.add);
     router.get ('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.get);
     router.post('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.add);
-    router.put ('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.update);
     router.del ('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.del);
     router.get ('/api/v1/oauth/clients/:clientId/tokens', settingsScope, routes.clients.getClientTokens);
     router.del ('/api/v1/oauth/clients/:clientId/tokens', settingsScope, routes.clients.delClientTokens);
@@ -144,7 +134,7 @@ function initializeExpressSync() {
     // app routes
     router.get ('/api/v1/apps',          appsScope, routes.apps.getApps);
     router.get ('/api/v1/apps/:id',      appsScope, routes.apps.getApp);
-    router.get ('/api/v1/apps/:id/icon', appsScope, routes.apps.getAppIcon);
+    router.get ('/api/v1/apps/:id/icon', routes.apps.getAppIcon);
 
     router.post('/api/v1/apps/install',       appsScope, routes.user.requireAdmin, routes.apps.installApp);
     router.post('/api/v1/apps/:id/uninstall', appsScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.apps.uninstallApp);
@@ -168,6 +158,12 @@ function initializeExpressSync() {
     router.post('/api/v1/settings/cloudron_name',      settingsScope, routes.settings.setCloudronName);
     router.get ('/api/v1/settings/cloudron_avatar',    settingsScope, routes.settings.getCloudronAvatar);
     router.post('/api/v1/settings/cloudron_avatar',    settingsScope, multipart, routes.settings.setCloudronAvatar);
+    router.get ('/api/v1/settings/dns_config',         settingsScope, routes.settings.getDnsConfig);
+    router.post('/api/v1/settings/dns_config',         settingsScope, routes.settings.setDnsConfig);
+    router.get ('/api/v1/settings/backup_config',      settingsScope, routes.settings.getBackupConfig);
+    router.post('/api/v1/settings/backup_config',      settingsScope, routes.settings.setBackupConfig);
+    router.post('/api/v1/settings/certificate',        settingsScope, routes.settings.setCertificate);
+    router.post('/api/v1/settings/admin_certificate',  settingsScope, routes.settings.setAdminCertificate);
 
     // backup routes
     router.get ('/api/v1/backups', settingsScope, routes.backups.get);
@@ -199,6 +195,7 @@ function initializeExpressSync() {
     return httpServer;
 }
 
+// provides hooks for the 'installer'
 function initializeInternalExpressSync() {
     var app = express();
     var httpServer = http.createServer(app);
@@ -209,7 +206,7 @@ function initializeInternalExpressSync() {
     var json = middleware.json({ strict: true, limit: QUERY_LIMIT }), // application/json
         urlencoded = middleware.urlencoded({ extended: false, limit: QUERY_LIMIT }); // application/x-www-form-urlencoded
 
-    app.use(middleware.morgan('dev', { immediate: false }));
+    if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('Box Internal :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
 
     var router = new express.Router();
     router.del = router.delete; // amend router.del for readability further on
@@ -237,8 +234,8 @@ function start(callback) {
     async.series([
         auth.initialize,
         database.initialize,
+        cloudron.initialize, // keep this here because it reads activation state that others depend on
         taskmanager.initialize,
-        cloudron.initialize,
         mailer.initialize,
         cron.initialize,
         gHttpServer.listen.bind(gHttpServer, config.get('port'), '127.0.0.1'),

src/settings.js

@@ -20,40 +20,60 @@ exports = module.exports = {
     getDeveloperMode: getDeveloperMode,
     setDeveloperMode: setDeveloperMode,
 
+    getDnsConfig: getDnsConfig,
+    setDnsConfig: setDnsConfig,
+
+    getBackupConfig: getBackupConfig,
+    setBackupConfig: setBackupConfig,
+
     getDefaultSync: getDefaultSync,
     getAll: getAll,
 
+    validateCertificate: validateCertificate,
+    setCertificate: setCertificate,
+    setAdminCertificate: setAdminCertificate,
+
     AUTOUPDATE_PATTERN_KEY: 'autoupdate_pattern',
     TIME_ZONE_KEY: 'time_zone',
     CLOUDRON_NAME_KEY: 'cloudron_name',
     DEVELOPER_MODE_KEY: 'developer_mode',
+    DNS_CONFIG_KEY: 'dns_config',
+    BACKUP_CONFIG_KEY: 'backup_config',
 
     events: new (require('events').EventEmitter)()
 };
 
 var assert = require('assert'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     CronJob = require('cron').CronJob,
     DatabaseError = require('./databaseerror.js'),
+    ejs = require('ejs'),
+    fs = require('fs'),
+    path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settingsdb = require('./settingsdb.js'),
+    shell = require('./shell.js'),
     util = require('util'),
+    x509 = require('x509'),
     _ = require('underscore');
 
 var gDefaults = (function () {
-    var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
-    tz = tz ? tz.trim() : 'America/Los_Angeles';
-
     var result = { };
     result[exports.AUTOUPDATE_PATTERN_KEY] = '00 00 1,3,5,23 * * *';
-    result[exports.TIME_ZONE_KEY] = tz;
+    result[exports.TIME_ZONE_KEY] = 'America/Los_Angeles';
     result[exports.CLOUDRON_NAME_KEY] = 'Cloudron';
     result[exports.DEVELOPER_MODE_KEY] = false;
+    result[exports.DNS_CONFIG_KEY] = { };
+    result[exports.BACKUP_CONFIG_KEY] = { };
 
     return result;
 })();
 
+var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
+    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
+
 if (config.TEST) {
     // avoid noisy warnings during npm test
     exports.events.setMaxListeners(100);
@@ -81,6 +101,7 @@ util.inherits(SettingsError, Error);
 SettingsError.INTERNAL_ERROR = 'Internal Error';
 SettingsError.NOT_FOUND = 'Not Found';
 SettingsError.BAD_FIELD = 'Bad Field';
+SettingsError.INVALID_CERT = 'Invalid certificate';
 
 function setAutoupdatePattern(pattern, callback) {
     assert.strictEqual(typeof pattern, 'string');
@@ -210,6 +231,79 @@ function setDeveloperMode(enabled, callback) {
     });
 }
 
+function getDnsConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settingsdb.get(exports.DNS_CONFIG_KEY, function (error, value) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.DNS_CONFIG_KEY]);
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        callback(null, JSON.parse(value)); // accessKeyId, secretAccessKey, region
+    });
+}
+
+function setDnsConfig(dnsConfig, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var credentials;
+
+    if (dnsConfig.provider === 'route53') {
+        if (typeof dnsConfig.accessKeyId !== 'string') return callback(new SettingsError(SettingsError.BAD_FIELD, 'accessKeyId must be a string'));
+        if (typeof dnsConfig.secretAccessKey !== 'string') return callback(new SettingsError(SettingsError.BAD_FIELD, 'secretAccessKey must be a string'));
+
+        credentials = {
+            provider: dnsConfig.provider,
+            accessKeyId: dnsConfig.accessKeyId,
+            secretAccessKey: dnsConfig.secretAccessKey,
+            region: dnsConfig.region || 'us-east-1',
+            endpoint: dnsConfig.endpoint || null
+        };
+    } else if (dnsConfig.provider === 'caas') {
+        credentials = {
+            provider: dnsConfig.provider
+        };
+    } else {
+        return callback(new SettingsError(SettingsError.BAD_FIELD, 'provider must be route53 or caas'));
+    }
+
+    settingsdb.set(exports.DNS_CONFIG_KEY, JSON.stringify(credentials), function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        exports.events.emit(exports.DNS_CONFIG_KEY, dnsConfig);
+
+        callback(null);
+    });
+}
+
+function getBackupConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settingsdb.get(exports.BACKUP_CONFIG_KEY, function (error, value) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.BACKUP_CONFIG_KEY]);
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        callback(null, JSON.parse(value)); // provider, token, key, region, prefix, bucket
+    });
+}
+
+function setBackupConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (backupConfig.provider !== 'caas') {
+        return callback(new SettingsError(SettingsError.BAD_FIELD, 'provider must be caas'));
+    }
+
+    settingsdb.set(exports.BACKUP_CONFIG_KEY, JSON.stringify(backupConfig), function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        exports.events.emit(exports.BACKUP_CONFIG_KEY, backupConfig);
+
+        callback(null);
+    });
+}
+
 function getDefaultSync(name) {
     assert.strictEqual(typeof name, 'string');
 
@@ -228,3 +322,104 @@ function getAll(callback) {
         callback(null, result);
     });
 }
+
+// note: https://tools.ietf.org/html/rfc4346#section-7.4.2 (certificate_list) requires that the
+// servers certificate appears first (and not the intermediate cert)
+function validateCertificate(cert, key, fqdn) {
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof fqdn, 'string');
+
+    if (cert === null && key === null) return null;
+    if (!cert && key) return new Error('missing cert');
+    if (cert && !key) return new Error('missing key');
+
+    var content;
+    try {
+        content = x509.parseCert(cert);
+    } catch (e) {
+        return new Error('invalid cert: ' + e.message);
+    }
+
+    // check expiration
+    if (content.notAfter < new Date()) return new Error('cert expired');
+
+    function matchesDomain(domain) {
+        if (domain === fqdn) return true;
+        if (domain.indexOf('*') === 0 && domain.slice(2) === fqdn.slice(fqdn.indexOf('.') + 1)) return true;
+
+        return false;
+    }
+
+    // check domain
+    var domains = content.altNames.concat(content.subject.commonName);
+    if (!domains.some(matchesDomain)) return new Error(util.format('cert is not valid for this domain. Expecting %s in %j', fqdn, domains));
+
+    // http://httpd.apache.org/docs/2.0/ssl/ssl_faq.html#verify
+    var certModulus = safe.child_process.execSync('openssl x509 -noout -modulus', { encoding: 'utf8', input: cert });
+    var keyModulus = safe.child_process.execSync('openssl rsa -noout -modulus', { encoding: 'utf8', input: key });
+    if (certModulus !== keyModulus) return new Error('key does not match the cert');
+
+    return null;
+}
+
+function setCertificate(cert, key, callback) {
+    assert.strictEqual(typeof cert, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateCertificate(cert, key, '*.' + config.fqdn());
+    if (error) return callback(new SettingsError(SettingsError.INVALID_CERT, error.message));
+
+    // backup the cert
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.cert'), cert)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.key'), key)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+
+    // copy over fallback cert
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), cert)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+
+    shell.sudo('setCertificate', [ RELOAD_NGINX_CMD ], function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function setAdminCertificate(cert, key, callback) {
+    assert.strictEqual(typeof cert, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var sourceDir = path.resolve(__dirname, '..');
+    var endpoint = 'admin';
+    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
+    var certFilePath = path.join(paths.APP_CERTS_DIR, 'admin.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, 'admin.key');
+
+    var error = validateCertificate(cert, key, vhost);
+    if (error) return callback(new SettingsError(SettingsError.INVALID_CERT, error.message));
+
+    // backup the cert
+    if (!safe.fs.writeFileSync(certFilePath, cert)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(keyFilePath, key)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+
+    var data = {
+        sourceDir: sourceDir,
+        adminOrigin: config.adminOrigin(),
+        vhost: vhost,
+        endpoint: endpoint,
+        certFilePath: certFilePath,
+        keyFilePath: keyFilePath
+    };
+    var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
+    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, 'admin.conf');
+
+    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) return callback(safe.error);
+
+    shell.sudo('setAdminCertificate', [ RELOAD_NGINX_CMD ], function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}

src/simpleauth.js

@@ -0,0 +1,155 @@
+'use strict';
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var apps = require('./apps.js'),
+    AppsError = apps.AppsError,
+    assert = require('assert'),
+    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
+    ClientsError = clients.ClientsError,
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:src/simpleauth'),
+    express = require('express'),
+    http = require('http'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    middleware = require('./middleware'),
+    tokendb = require('./tokendb.js'),
+    user = require('./user.js'),
+    UserError = require('./user.js').UserError;
+
+var gHttpServer = null;
+
+function loginLogic(clientId, username, password, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof username, 'string');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('login: client %s and user %s', clientId, username);
+
+    clients.get(clientId, function (error, clientObject) {
+        if (error) return callback(error);
+
+        // only allow simple auth clients
+        if (clientObject.type !== clientdb.TYPE_SIMPLE_AUTH) return callback(new ClientsError(ClientsError.INVALID_CLIENT));
+
+        user.verify(username, password, function (error, userObject) {
+            if (error) return callback(error);
+
+            apps.get(clientObject.appId, function (error, appObject) {
+                if (error) return callback(error);
+
+                if (!apps.hasAccessTo(appObject, userObject)) return callback(new AppsError(AppsError.ACCESS_DENIED));
+
+                var accessToken = tokendb.generateToken();
+                var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+
+                tokendb.add(accessToken, tokendb.PREFIX_USER + userObject.id, clientId, expires, clientObject.scope, function (error) {
+                    if (error) return callback(error);
+
+                    debug('login: new access token for client %s and user %s: %s', clientId, username, accessToken);
+
+                    callback(null, { accessToken: accessToken, user: userObject });
+                });
+            });
+        });
+    });
+}
+
+function logoutLogic(accessToken, callback) {
+    assert.strictEqual(typeof accessToken, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('logout: %s', accessToken);
+
+    tokendb.del(accessToken, function (error) {
+        if (error) return callback(error);
+        callback(null);
+    });
+}
+
+function login(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.clientId !== 'string') return next(new HttpError(400, 'clientId is required'));
+    if (typeof req.body.username !== 'string') return next(new HttpError(400, 'username is required'));
+    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'password is required'));
+
+    loginLogic(req.body.clientId, req.body.username, req.body.password, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(401, 'Unknown client'));
+        if (error && error.reason === ClientsError.INVALID_CLIENT) return next(new HttpError(401, 'Unkown client'));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(401, 'Forbidden'));
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(401, 'Unkown app'));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new HttpError(401, 'Forbidden'));
+        if (error && error.reason === AppsError.ACCESS_DENIED) return next(new HttpError(401, 'Forbidden'));
+        if (error) return next(new HttpError(500, error));
+
+        var tmp = {
+            accessToken: result.accessToken,
+            user: {
+                id: result.user.id,
+                username: result.user.username,
+                email: result.user.email,
+                admin: !!result.user.admin
+            }
+        };
+
+        next(new HttpSuccess(200, tmp));
+    });
+}
+
+function logout(req, res, next) {
+    assert.strictEqual(typeof req.query, 'object');
+
+    if (typeof req.query.access_token !== 'string') return next(new HttpError(400, 'access_token in query required'));
+
+    logoutLogic(req.query.access_token, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(401, 'Forbidden'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, {}));
+    });
+}
+
+function initializeExpressSync() {
+    var app = express();
+    var httpServer = http.createServer(app);
+
+    httpServer.on('error', console.error);
+
+    var json = middleware.json({ strict: true, limit: '100kb' });
+    var router = new express.Router();
+
+    // basic auth
+    router.post('/api/v1/login', login);
+    router.get ('/api/v1/logout', logout);
+
+    if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('SimpleAuth :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
+
+    app
+        .use(middleware.timeout(10000))
+        .use(json)
+        .use(router)
+        .use(middleware.lastMile());
+
+    return httpServer;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gHttpServer = initializeExpressSync();
+    gHttpServer.listen(config.get('simpleAuthPort'), '0.0.0.0', callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gHttpServer.close(callback);
+}

src/storage/caas.js

@@ -0,0 +1,129 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl,
+
+    copyObject: copyObject,
+
+    getAllPaged: getAllPaged
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('../config.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function getBackupCredentials(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+    assert(backupConfig.token);
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
+    superagent.post(url).query({ token: backupConfig.token }).end(function (error, result) {
+        if (error) return callback(error);
+        if (result.statusCode !== 201) return callback(new Error(result.text));
+        if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
+
+        var credentials = {
+            accessKeyId: result.body.credentials.AccessKeyId,
+            secretAccessKey: result.body.credentials.SecretAccessKey,
+            sessionToken: result.body.credentials.SessionToken,
+            region: 'us-east-1'
+        };
+
+        if (backupConfig.endpoint) credentials.endpoint = new AWS.Endpoint(backupConfig.endpoint);
+
+        callback(null, credentials);
+    });
+}
+
+function getAllPaged(backupConfig, page, perPage, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+    superagent.get(url).query({ token: backupConfig.token }).end(function (error, result) {
+        if (error) return callback(error);
+        if (result.statusCode !== 200) return callback(new Error(result.text));
+        if (!result.body || !util.isArray(result.body.backups)) return callback(new Error('Unexpected response'));
+
+        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
+        return callback(null, result.body.backups);
+    });
+}
+
+function getSignedUploadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!backupConfig.bucket || !backupConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!backupConfig.bucket || !backupConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function copyObject(backupConfig, from, to, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof from, 'string');
+    assert.strictEqual(typeof to, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!backupConfig.bucket || !backupConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        var params = {
+            Bucket: backupConfig.bucket, // target bucket
+            Key: backupConfig.prefix + '/' + to, // target file
+            CopySource: backupConfig.bucket + '/' + backupConfig.prefix + '/' + from, // source
+        };
+
+        var s3 = new AWS.S3(credentials);
+        s3.copyObject(params, callback);
+    });
+}

src/storage/s3.js

@@ -0,0 +1,104 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl,
+
+    copyObject: copyObject,
+    getAllPaged: getAllPaged
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk');
+
+function getBackupCredentials(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    assert(backupConfig.accessKeyId && backupConfig.secretAccessKey);
+
+    var credentials = {
+        accessKeyId: backupConfig.accessKeyId,
+        secretAccessKey: backupConfig.secretAccessKey,
+        region: 'us-east-1'
+    };
+
+    if (backupConfig.endpoint) credentials.endpoint = new AWS.Endpoint(backupConfig.endpoint);
+
+    callback(null, credentials);
+}
+
+function getAllPaged(backupConfig, page, perPage, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    return callback(new Error('Not implemented yet'));
+}
+
+function getSignedUploadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function copyObject(backupConfig, from, to, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof from, 'string');
+    assert.strictEqual(typeof to, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
+        if (error) return callback(error);
+
+        var params = {
+            Bucket: backupConfig.bucket, // target bucket
+            Key: backupConfig.prefix + '/' + to, // target file
+            CopySource: backupConfig.bucket + '/' + backupConfig.prefix + '/' + from, // source
+        };
+
+        var s3 = new AWS.S3(credentials);
+        s3.copyObject(params, callback);
+    });
+}

src/subdomains.js

@@ -0,0 +1,137 @@
+/* jslint node:true */
+
+'use strict';
+
+module.exports = exports = {
+    add: add,
+    remove: remove,
+    status: status,
+    update: update, // unlike add, this fetches latest value, compares and adds if necessary. atomicity depends on backend
+    get: get,
+
+    SubdomainError: SubdomainError
+};
+
+var assert = require('assert'),
+    caas = require('./dns/caas.js'),
+    config = require('./config.js'),
+    route53 = require('./dns/route53.js'),
+    settings = require('./settings.js'),
+    util = require('util');
+
+function SubdomainError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(SubdomainError, Error);
+
+SubdomainError.NOT_FOUND = 'No such domain';
+SubdomainError.EXTERNAL_ERROR = 'External error';
+SubdomainError.STILL_BUSY = 'Still busy';
+SubdomainError.MISSING_CREDENTIALS = 'Missing credentials';
+SubdomainError.INTERNAL_ERROR = 'Missing credentials';
+
+// choose which subdomain backend we use for test purpose we use route53
+function api(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    switch (provider) {
+        case 'caas': return caas;
+        case 'route53': return route53;
+        default: return null;
+    }
+}
+
+function add(subdomain, type, values, callback) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).add(dnsConfig, config.zoneName(), subdomain, type, values, function (error, changeId) {
+            if (error) return callback(error);
+            callback(null, changeId);
+        });
+    });
+}
+
+function get(subdomain, type, callback) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).get(dnsConfig, config.zoneName(), subdomain, type, function (error, values) {
+            if (error) return callback(error);
+
+            callback(null, values);
+        });
+    });
+}
+
+function update(subdomain, type, values, callback) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).update(dnsConfig, config.zoneName(), subdomain, type, values, function (error) {
+            if (error) return callback(error);
+
+            callback(null);
+        });
+    });
+}
+
+function remove(subdomain, type, values, callback) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).del(dnsConfig, config.zoneName(), subdomain, type, values, function (error) {
+            if (error && error.reason !== SubdomainError.NOT_FOUND) return callback(error);
+
+            callback(null);
+        });
+    });
+}
+
+function status(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).getChangeStatus(dnsConfig, changeId, function (error, status) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error));
+            callback(null, status === 'INSYNC' ? 'done' : 'pending');
+        });
+    });
+}

src/taskmanager.js

@@ -9,7 +9,9 @@ exports = module.exports = {
 
 var appdb = require('./appdb.js'),
     assert = require('assert'),
+    async = require('async'),
     child_process = require('child_process'),
+    cloudron = require('./cloudron.js'),
     debug = require('debug')('box:taskmanager'),
     locker = require('./locker.js'),
     _ = require('underscore');
@@ -17,44 +19,57 @@ var appdb = require('./appdb.js'),
 var gActiveTasks = { };
 var gPendingTasks = [ ];
 
-// Task concurrency is 1 for two reasons:
-// 1. The backup scripts (app and box) turn off swap after finish disregarding other backup processes
-// 2. apptask getFreePort has race with multiprocess
-var TASK_CONCURRENCY = 1;
-var NOOP_CALLBACK = function (error) { console.error(error); };
+var TASK_CONCURRENCY = 5;
+var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    // resume app installs and uninstalls
-    appdb.getAll(function (error, apps) {
-        if (error) return callback(error);
-
-        apps.forEach(function (app) {
-            debug('Creating process for %s (%s) with state %s', app.location, app.id, app.installationState);
-            startAppTask(app.id);
-        });
-
-        callback(null);
-    });
-
     locker.on('unlocked', startNextTask);
+
+    if (cloudron.isConfiguredSync()) {
+        resumeTasks();
+    } else {
+        cloudron.events.on(cloudron.EVENT_CONFIGURED, resumeTasks);
+    }
+
+    callback();
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     gPendingTasks = [ ]; // clear this first, otherwise stopAppTask will resume them
-    for (var appId in gActiveTasks) {
-        stopAppTask(appId);
-    }
 
-    callback(null);
+    cloudron.events.removeListener(cloudron.EVENT_CONFIGURED, resumeTasks);
+    locker.removeListener('unlocked', startNextTask);
+
+    async.eachSeries(Object.keys(gActiveTasks), stopAppTask, callback);
+}
+
+
+// resume app installs and uninstalls
+function resumeTasks(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(error);
+
+        apps.forEach(function (app) {
+            if (app.installationState === appdb.ISTATE_INSTALLED && app.runState === appdb.RSTATE_RUNNING) return;
+
+            debug('Creating process for %s (%s) with state %s', app.location, app.id, app.installationState);
+            startAppTask(app.id);
+        });
+
+        callback(null);
+    });
 }
 
 function startNextTask() {
     if (gPendingTasks.length === 0) return;
-    assert.strictEqual(Object.keys(gActiveTasks).length, 0); // since we allow only one task at a time
+
+    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     startAppTask(gPendingTasks.shift());
 }
@@ -63,17 +78,27 @@ function startAppTask(appId) {
     assert.strictEqual(typeof appId, 'string');
     assert(!(appId in gActiveTasks));
 
-    var lockError = locker.lock(locker.OP_APPTASK);
-
-    if (lockError || Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
+    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug('Reached concurrency limit, queueing task for %s', appId);
         gPendingTasks.push(appId);
         return;
     }
 
+    var lockError = locker.recursiveLock(locker.OP_APPTASK);
+
+    if (lockError) {
+        debug('Locked for another operation, queueing task for %s', appId);
+        gPendingTasks.push(appId);
+        return;
+    }
+
     gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
+
+    var pid = gActiveTasks[appId].pid;
+    debug('Started task of %s pid: %s', appId, pid);
+
     gActiveTasks[appId].once('exit', function (code) {
-        debug('Task for %s completed with status %s', appId, code);
+        debug('Task for %s pid %s completed with status %s', appId, pid, code);
         if (code && code !== 50) { // apptask crashed
             appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code }, NOOP_CALLBACK);
         }
@@ -82,21 +107,32 @@ function startAppTask(appId) {
     });
 }
 
-function stopAppTask(appId) {
+function stopAppTask(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     if (gActiveTasks[appId]) {
-        debug('stopAppTask : Killing existing task of %s with pid %s: ', appId, gActiveTasks[appId].pid);
+        debug('stopAppTask : Killing existing task of %s with pid %s', appId, gActiveTasks[appId].pid);
+        gActiveTasks[appId].once('exit', function () { callback(); });
         gActiveTasks[appId].kill(); // this will end up calling the 'exit' handler
-        delete gActiveTasks[appId];
-    } else if (gPendingTasks.indexOf(appId) !== -1) {
-        debug('stopAppTask: Removing existing pending task : %s', appId);
-        gPendingTasks = _.without(gPendingTasks, appId);
+        return;
     }
+
+    if (gPendingTasks.indexOf(appId) !== -1) {
+        debug('stopAppTask: Removing pending task : %s', appId);
+        gPendingTasks = _.without(gPendingTasks, appId);
+    } else {
+        debug('stopAppTask: no task for %s to be stopped', appId);
+    }
+
+    callback();
 }
 
-function restartAppTask(appId) {
-    stopAppTask(appId);
-    startAppTask(appId);
-}
+function restartAppTask(appId, callback) {
+    callback = callback || NOOP_CALLBACK;
 
+    async.series([
+        stopAppTask.bind(null, appId),
+        startAppTask.bind(null, appId)
+    ], callback);
+}

src/test/apps-test.js

@@ -36,14 +36,15 @@ describe('Apps', function () {
         containerId: null,
         portBindings: { PORT: 5678 },
         healthy: null,
-        accessRestriction: ''
+        accessRestriction: null,
+        oauthProxy: false
     };
 
     before(function (done) {
         async.series([
             database.initialize,
             database._clear,
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.oauthProxy)
         ], done);
     });
 
@@ -157,5 +158,48 @@ describe('Apps', function () {
             });
         });
     });
-});
 
+    describe('validateAccessRestriction', function () {
+        it('allows null input', function () {
+            expect(apps._validateAccessRestriction(null)).to.eql(null);
+        });
+
+        it('does not allow wrong user type', function () {
+            expect(apps._validateAccessRestriction({ users: {} })).to.be.an(Error);
+        });
+
+        it('does not allow no user input', function () {
+            expect(apps._validateAccessRestriction({ users: [] })).to.be.an(Error);
+        });
+
+        it('allows single user input', function () {
+            expect(apps._validateAccessRestriction({ users: [ 'someuserid' ] })).to.eql(null);
+        });
+
+        it('allows multi user input', function () {
+            expect(apps._validateAccessRestriction({ users: [ 'someuserid', 'someuserid1', 'someuserid2', 'someuserid3' ] })).to.eql(null);
+        });
+    });
+
+    describe('hasAccessTo', function () {
+        it('returns true for unrestricted access', function () {
+            expect(apps.hasAccessTo({ accessRestriction: null }, { id: 'someuser' })).to.equal(true);
+        });
+
+        it('returns true for allowed user', function () {
+            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'someuser' ] } }, { id: 'someuser' })).to.equal(true);
+        });
+
+        it('returns true for allowed user with multiple allowed', function () {
+            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'someuser', 'anotheruser' ] } }, { id: 'someuser' })).to.equal(true);
+        });
+
+        it('returns false for not allowed user', function () {
+            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'foo' ] } }, { id: 'someuser' })).to.equal(false);
+        });
+
+        it('returns false for not allowed user with multiple allowed', function () {
+            expect(apps.hasAccessTo({ accessRestriction: { users: [ 'foo', 'anotheruser' ] } }, { id: 'someuser' })).to.equal(false);
+        });
+    });
+});

src/test/apptask-test.js

@@ -9,14 +9,16 @@
 var addons = require('../addons.js'),
     appdb = require('../appdb.js'),
     apptask = require('../apptask.js'),
+    async = require('async'),
     config = require('../config.js'),
     database = require('../database.js'),
     expect = require('expect.js'),
     fs = require('fs'),
+    js2xml = require('js2xmlparser'),
     net = require('net'),
     nock = require('nock'),
     paths = require('../paths.js'),
-    sysinfo = require('../sysinfo.js'),
+    settings = require('../settings.js'),
     _ = require('underscore');
 
 var MANIFEST = {
@@ -29,7 +31,7 @@ var MANIFEST = {
   "contactEmail": "support@cloudron.io",
   "version": "0.1.0",
   "manifestVersion": 1,
-  "dockerImage": "girish/test:0.2.0",
+  "dockerImage": "cloudron/test:8.0.0",
   "healthCheckPath": "/",
   "httpPort": 7777,
   "tcpPorts": {
@@ -57,17 +59,34 @@ var APP = {
     containerId: null,
     httpPort: 4567,
     portBindings: null,
-    accessRestriction: '',
+    accessRestriction: null,
+    oauthProxy: false,
     dnsRecordId: 'someDnsRecordId'
 };
 
+ var awsHostedZones = {
+     HostedZones: [{
+         Id: '/hostedzone/ZONEID',
+         Name: 'localhost.',
+         CallerReference: '305AFD59-9D73-4502-B020-F4E6F889CB30',
+         ResourceRecordSetCount: 2,
+         ChangeInfo: {
+             Id: '/change/CKRTFJA0ANHXB',
+             Status: 'INSYNC'
+         }
+     }],
+    IsTruncated: false,
+    MaxItems: '100'
+ };
+
 describe('apptask', function () {
     before(function (done) {
         config.set('version', '0.5.0');
-        database.initialize(function (error) {
-            expect(error).to.be(null);
-            appdb.add(APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, done);
-        });
+        async.series([
+            database.initialize,
+            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, APP.oauthProxy),
+            settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' })
+        ], done);
     });
 
     after(function (done) {
@@ -121,21 +140,21 @@ describe('apptask', function () {
     });
 
     it('allocate OAuth credentials', function (done) {
-        addons._allocateOAuthCredentials(APP, function (error) {
+        addons._setupOauth(APP, {}, function (error) {
             expect(error).to.be(null);
             done();
         });
     });
 
     it('remove OAuth credentials', function (done) {
-        addons._removeOAuthCredentials(APP, function (error) {
+        addons._teardownOauth(APP, {}, function (error) {
             expect(error).to.be(null);
             done();
         });
     });
 
     it('remove OAuth credentials twice succeeds', function (done) {
-        addons._removeOAuthCredentials(APP, function (error) {
+        addons._teardownOauth(APP, {}, function (error) {
             expect(!error).to.be.ok();
             done();
         });
@@ -154,7 +173,7 @@ describe('apptask', function () {
     it('barfs on bad manifest', function (done) {
         var badApp = _.extend({ }, APP);
         badApp.manifest = _.extend({ }, APP.manifest);
-        delete badApp.manifest['id'];
+        delete badApp.manifest.id;
 
         apptask._verifyManifest(badApp, function (error) {
             expect(error).to.be.ok();
@@ -184,26 +203,32 @@ describe('apptask', function () {
 
     it('registers subdomain', function (done) {
         nock.cleanAll();
-        var scope = nock(config.apiServerOrigin())
-            .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP.location, type: 'A', value: sysinfo.getIp() } ] })
-            .reply(201, { ids: [ APP.dnsRecordId ] });
+
+        var awsScope = nock('http://localhost:5353')
+            .get('/2013-04-01/hostedzone')
+            .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }))
+            .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+            .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'RRID', Status: 'INSYNC' } }));
 
         apptask._registerSubdomain(APP, function (error) {
             expect(error).to.be(null);
-            expect(scope.isDone()).to.be.ok();
+            expect(awsScope.isDone()).to.be.ok();
             done();
         });
     });
 
     it('unregisters subdomain', function (done) {
         nock.cleanAll();
-        var scope = nock(config.apiServerOrigin())
-            .delete('/api/v1/subdomains/' + APP.dnsRecordId + '?token=' + config.token())
-            .reply(204, {});
 
-        apptask._unregisterSubdomain(APP, function (error) {
+        var awsScope = nock('http://localhost:5353')
+            .get('/2013-04-01/hostedzone')
+            .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }))
+            .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+            .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'RRID', Status: 'INSYNC' } }));
+
+        apptask._unregisterSubdomain(APP, APP.location, function (error) {
             expect(error).to.be(null);
-            expect(scope.isDone()).to.be.ok();
+            expect(awsScope.isDone()).to.be.ok();
             done();
         });
     });