better debugs

This ensures that
a) we don't get emails from janitor about bad app backups
b) that the backups are persisted over the s3 lifecycle

Fixes #493

.gitignore

@@ -4,10 +4,6 @@ docs/
 webadmin/dist/
 setup/splash/website/
 
-# vim swam files
+# vim swap files
 *.swp
 
-# supervisor
-supervisord.pid
-supervisord.log
-

README.md

@@ -4,7 +4,6 @@ The Box
 Development setup
 -----------------
 * sudo useradd -m yellowtent
-** This dummy user is required for supervisor 'box' configs
 ** Add admin-localhost as 127.0.0.1 in /etc/hosts
 ** All apps will be installed as hypened-subdomains of localhost. You should add
    hyphened-subdomains of your apps into /etc/hosts

app.js

@@ -1,47 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var server = require('./src/server.js'),
-    ldap = require('./src/ldap.js'),
-    config = require('./src/config.js');
-
-console.log();
-console.log('==========================================');
-console.log(' Cloudron will use the following settings ');
-console.log('==========================================');
-console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
-console.log(' Version:                        ', config.version());
-console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore token:                 ', config.token());
-console.log(' Appstore API server origin:     ', config.apiServerOrigin());
-console.log(' Appstore Web server origin:     ', config.webServerOrigin());
-console.log();
-console.log('==========================================');
-console.log();
-
-server.start(function (err) {
-    if (err) {
-        console.error('Error starting server', err);
-        process.exit(1);
-    }
-
-    console.log('Server listening on port ' + config.get('port'));
-
-    ldap.start(function (error) {
-        if (error) {
-            console.error('Error LDAP starting server', err);
-            process.exit(1);
-        }
-
-        console.log('LDAP server listen on port ' + config.get('ldapPort'));
-    });
-});
-
-var NOOP_CALLBACK = function () { };
-
-process.on('SIGINT', function () { server.stop(NOOP_CALLBACK); });
-process.on('SIGTERM', function () { server.stop(NOOP_CALLBACK); });

box.js

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
+var appHealthMonitor = require('./src/apphealthmonitor.js'),
+    async = require('async'),
+    config = require('./src/config.js'),
+    ldap = require('./src/ldap.js'),
+    oauthproxy = require('./src/oauthproxy.js'),
+    server = require('./src/server.js');
+
+console.log();
+console.log('==========================================');
+console.log(' Cloudron will use the following settings ');
+console.log('==========================================');
+console.log();
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
+console.log(' Version:                        ', config.version());
+console.log(' Admin Origin:                   ', config.adminOrigin());
+console.log(' Appstore token:                 ', config.token());
+console.log(' Appstore API server origin:     ', config.apiServerOrigin());
+console.log(' Appstore Web server origin:     ', config.webServerOrigin());
+console.log();
+console.log('==========================================');
+console.log();
+
+async.series([
+    server.start,
+    ldap.start,
+    appHealthMonitor.start,
+    oauthproxy.start
+], function (error) {
+    if (error) {
+        console.error('Error starting server', error);
+        process.exit(1);
+    }
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});

crashnotifier.js

@@ -2,20 +2,12 @@
 
 'use strict';
 
-// WARNING This is a supervisor eventlistener!
-//         The communication happens via stdin/stdout
-//         !! No console.log() allowed
-//         !! Do not set DEBUG
-
 var assert = require('assert'),
     mailer = require('./src/mailer.js'),
     safe = require('safetydance'),
-    supervisor = require('supervisord-eventlistener'),
     path = require('path'),
     util = require('util');
 
-var gLastNotifyTime = {};
-var gCooldownTime = 1000 * 60  * 5; // 5 min
 var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
 
 function collectLogs(program, callback) {
@@ -26,28 +18,30 @@ function collectLogs(program, callback) {
     callback(null, logs);
 }
 
-supervisor.on('PROCESS_STATE_EXITED', function (headers, data) {
-    if (data.expected === '1') return console.error('Normal app %s exit', data.processname);
-
-    console.error('%s exited unexpectedly', data.processname);
-
-    collectLogs(data.processname, function (error, result) {
+function sendCrashNotification(processName) {
+    collectLogs(processName, function (error, result) {
         if (error) {
             console.error('Failed to collect logs.', error);
             result = util.format('Failed to collect logs.', error);
         }
 
-        if (!gLastNotifyTime[data.processname] || gLastNotifyTime[data.processname] < Date.now() - gCooldownTime) {
-            console.error('Send mail.');
-            mailer.sendCrashNotification(data.processname, result);
-            gLastNotifyTime[data.processname] = Date.now();
-        } else {
-            console.error('Do not send mail, already sent one recently.');
-        }
+        console.log('Sending crash notification email for', processName);
+        mailer.sendCrashNotification(processName, result);
     });
-});
+}
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    mailer.initialize(function (error) {
+        if (error) return console.error(error);
+
+        sendCrashNotification(processName);
+    });
+}
+
+main();
 
-mailer.initialize(function () {
-    supervisor.listen(process.stdin, process.stdout);
-    console.error('Crashnotifier listening...');
-});

janitor.js

@@ -4,6 +4,12 @@
 
 require('supererror')({ splatchError: true });
 
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
 var assert = require('assert'),
     debug = require('debug')('box:janitor'),
     async = require('async'),
@@ -11,8 +17,6 @@ var assert = require('assert'),
     authcodedb = require('./src/authcodedb.js'),
     database = require('./src/database.js');
 
-var TOKEN_CLEANUP_INTERVAL = 30000;
-
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -52,7 +56,7 @@ function run() {
         cleanupExpiredAuthCodes(function (error) {
             if (error) console.error(error);
 
-            setTimeout(run, TOKEN_CLEANUP_INTERVAL);
+            process.exit(0);
         });
     });
 }

npm-shrinkwrap.json

@@ -7,6 +7,28 @@
       "from": "https://registry.npmjs.org/async/-/async-1.2.1.tgz",
       "resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz"
     },
+    "aws-sdk": {
+      "version": "2.1.46",
+      "from": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "dependencies": {
+        "sax": {
+          "version": "0.5.3",
+          "from": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz",
+          "resolved": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz"
+        },
+        "xml2js": {
+          "version": "0.2.8",
+          "from": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz",
+          "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz"
+        },
+        "xmlbuilder": {
+          "version": "0.4.2",
+          "from": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz",
+          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz"
+        }
+      }
+    },
     "body-parser": {
       "version": "1.13.1",
       "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.13.1.tgz",
@@ -105,23 +127,24 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "1.6.0",
-      "from": "cloudron-manifestformat@1.6.0",
+      "version": "1.7.0",
+      "from": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
       "dependencies": {
         "java-packagename-regex": {
           "version": "1.0.0",
-          "from": "java-packagename-regex@>=1.0.0 <2.0.0",
+          "from": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz",
           "resolved": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz"
         },
         "safetydance": {
           "version": "0.0.15",
-          "from": "safetydance@0.0.15",
+          "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz",
           "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz"
         },
         "tv4": {
-          "version": "1.1.12",
-          "from": "tv4@>=1.1.9 <2.0.0",
-          "resolved": "http://registry.npmjs.org/tv4/-/tv4-1.1.12.tgz"
+          "version": "1.2.3",
+          "from": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz",
+          "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz"
         }
       }
     },
@@ -2418,11 +2441,6 @@
         }
       }
     },
-    "supervisord-eventlistener": {
-      "version": "0.1.0",
-      "from": "https://registry.npmjs.org/supervisord-eventlistener/-/supervisord-eventlistener-0.1.0.tgz",
-      "resolved": "https://registry.npmjs.org/supervisord-eventlistener/-/supervisord-eventlistener-0.1.0.tgz"
-    },
     "tail-stream": {
       "version": "0.2.1",
       "from": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",

oauthproxy.js

@@ -1,185 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var express = require('express'),
-    url = require('url'),
-    uuid = require('node-uuid'),
-    async = require('async'),
-    superagent = require('superagent'),
-    assert = require('assert'),
-    debug = require('debug')('box:proxy'),
-    proxy = require('proxy-middleware'),
-    session = require('cookie-session'),
-    database = require('./src/database.js'),
-    appdb = require('./src/appdb.js'),
-    clientdb = require('./src/clientdb.js'),
-    config = require('./src/config.js'),
-    http = require('http');
-
-// Allow self signed certs!
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
-
-var gSessions = {};
-var gProxyMiddlewareCache = {};
-var gApp = express();
-var gHttpServer = http.createServer(gApp);
-
-var CALLBACK_URI = '/callback';
-var PORT = 4000;
-
-function startServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    gHttpServer.on('error', console.error);
-
-    gApp.use(session({
-        keys: ['blue', 'cheese', 'is', 'something']
-    }));
-
-    // ensure we have a in memory store for the session to cache client information
-    gApp.use(function (req, res, next) {
-        assert.strictEqual(typeof req.session, 'object');
-
-        if (!req.session.id || !gSessions[req.session.id]) {
-            req.session.id = uuid.v4();
-            gSessions[req.session.id] = {};
-        }
-
-        // attach the session data to the requeset
-        req.sessionData = gSessions[req.session.id];
-
-        next();
-    });
-
-    gApp.use(function verifySession(req, res, next) {
-        assert.strictEqual(typeof req.sessionData, 'object');
-
-        if (!req.sessionData.accessToken) {
-            req.authenticated = false;
-            return next();
-        }
-
-        superagent.get(config.adminOrigin() + '/api/v1/profile').query({ access_token: req.sessionData.accessToken}).end(function (error, result) {
-            if (error) {
-                console.error(error);
-                req.authenticated = false;
-            } else if (result.statusCode !== 200) {
-                req.sessionData.accessToken = null;
-                req.authenticated = false;
-            } else {
-                req.authenticated = true;
-            }
-
-            next();
-        });
-    });
-
-    gApp.use(function (req, res, next) {
-        // proceed if we are authenticated
-        if (req.authenticated) return next();
-
-        if (req.path === CALLBACK_URI && req.sessionData.returnTo) {
-            // exchange auth code for an access token
-            var query = {
-                response_type: 'token',
-                client_id: req.sessionData.clientId
-            };
-
-            var data = {
-                grant_type: 'authorization_code',
-                code: req.query.code,
-                redirect_uri: req.sessionData.returnTo,
-                client_id: req.sessionData.clientId,
-                client_secret: req.sessionData.clientSecret
-            };
-
-            superagent.post(config.adminOrigin() + '/api/v1/oauth/token').query(query).send(data).end(function (error, result) {
-                if (error) {
-                    console.error(error);
-                    return res.send(500, 'Unable to contact the oauth server.');
-                }
-                if (result.statusCode !== 200) {
-                    console.error('Failed to exchange auth code for a token.', result.statusCode, result.body);
-                    return res.send(500, 'Failed to exchange auth code for a token.');
-                }
-
-                req.sessionData.accessToken = result.body.access_token;
-
-                debug('user verified.');
-
-                // now redirect to the actual initially requested URL
-                res.redirect(req.sessionData.returnTo);
-            });
-        } else {
-            var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
-
-            if (!Number.isFinite(port)) {
-                console.error('Failed to parse nginx proxy header to get app port.');
-                return res.send(500, 'Routing error. No forwarded port.');
-            }
-
-            debug('begin verifying user for app on port %s.', port);
-
-            appdb.getByHttpPort(port, function (error, result) {
-                if (error) {
-                    console.error('Unknown app.', error);
-                    return res.send(500, 'Unknown app.');
-                }
-
-                clientdb.getByAppId('proxy-' + result.id, function (error, result) {
-                    if (error) {
-                        console.error('Unkonwn OAuth client.', error);
-                        return res.send(500, 'Unknown OAuth client.');
-                    }
-
-                    req.sessionData.port = port;
-                    req.sessionData.returnTo = result.redirectURI + req.path;
-                    req.sessionData.clientId = result.id;
-                    req.sessionData.clientSecret = result.clientSecret;
-
-                    var callbackUrl = result.redirectURI + CALLBACK_URI;
-                    var scope = 'profile,roleUser';
-                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + result.id + '&redirect_uri=' + callbackUrl + '&scope=' + scope;
-
-                    debug('begin OAuth flow for client %s.', result.name);
-
-                    // begin the OAuth flow
-                    res.redirect(oauthLogin);
-                });
-            });
-        }
-    });
-
-    gApp.use(function (req, res, next) {
-        var port = req.sessionData.port;
-
-        debug('proxy request for port %s with path %s.', port, req.path);
-
-        var proxyMiddleware = gProxyMiddlewareCache[port];
-        if (!proxyMiddleware) {
-            console.log('Adding proxy middleware for port %d', port);
-
-            proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
-            gProxyMiddlewareCache[port] = proxyMiddleware;
-        }
-
-        proxyMiddleware(req, res, next);
-    });
-
-    gHttpServer.listen(PORT, callback);
-}
-
-async.series([
-    database.initialize,
-    startServer
-], function (error) {
-    if (error) {
-        console.error('Failed to start proxy server.', error);
-        process.exit(1);
-    }
-
-    console.log('Proxy server listening...');
-});

package.json

@@ -17,8 +17,9 @@
   },
   "dependencies": {
     "async": "^1.2.1",
+    "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^1.6.0",
+    "cloudron-manifestformat": "^1.7.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
@@ -60,7 +61,6 @@
     "split": "^1.0.0",
     "superagent": "~0.21.0",
     "supererror": "^0.7.0",
-    "supervisord-eventlistener": "^0.1.0",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "underscore": "^1.7.0",
     "valid-url": "^1.0.9",
@@ -68,7 +68,6 @@
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "^2.1.10",
     "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",
@@ -83,6 +82,7 @@
     "gulp-uglify": "^1.1.0",
     "hock": "~1.2.0",
     "istanbul": "*",
+    "js2xmlparser": "^1.0.0",
     "mocha": "*",
     "nock": "^2.6.0",
     "node-sass": "^3.0.0-alpha.0",

setup/DESIGN.md

@@ -16,7 +16,7 @@ and replace it with a new one for an update.
 
 Because we do not package things as Docker yet, we should be careful
 about the code here. We have to expect remains of an older setup code.
-For example, older supervisor or nginx configs might be around.
+For example, older systemd or nginx configs might be around.
 
 The config directory is _part_ of the container and is not a VOLUME.
 Which is to say that the files will be nuked from one update to the next.
@@ -40,7 +40,7 @@ version (see below) or the mysql/postgresql data etc.
 
   * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
 
-  * supervisor is then started
+  * box services are then started
 
 setup_infra.sh
 This setups containers like graphite, mail and the addons containers.

setup/INFRA_VERSION

@@ -3,15 +3,15 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=8
+INFRA_VERSION=10
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.3.1
-MYSQL_IMAGE=cloudron/mysql:0.3.2
-POSTGRESQL_IMAGE=cloudron/postgresql:0.3.1
-MONGODB_IMAGE=cloudron/mongodb:0.3.1
-REDIS_IMAGE=cloudron/redis:0.3.1 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.3.1
-GRAPHITE_IMAGE=cloudron/graphite:0.3.3
+BASE_IMAGE=cloudron/base:0.3.3
+MYSQL_IMAGE=cloudron/mysql:0.3.3
+POSTGRESQL_IMAGE=cloudron/postgresql:0.3.2
+MONGODB_IMAGE=cloudron/mongodb:0.3.2
+REDIS_IMAGE=cloudron/redis:0.3.2 # if you change this, fix src/addons.js as well
+MAIL_IMAGE=cloudron/mail:0.3.2
+GRAPHITE_IMAGE=cloudron/graphite:0.3.4
 

setup/argparser.sh

@@ -16,6 +16,8 @@ arg_tls_key=""
 arg_token=""
 arg_version=""
 arg_web_server_origin=""
+arg_backup_key=""
+arg_aws=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -41,6 +43,12 @@ EOF
         arg_restore_key=$(echo "$2" | $json restoreKey)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
+        arg_backup_key=$(echo "$2" | $json backupKey)
+        [[ "${arg_backup_key}" == "null" ]] && arg_backup_key=""
+
+        arg_aws=$(echo "$2" | $json aws)
+        [[ "${arg_aws}" == "null" ]] && arg_aws=""
+
         shift 2
         ;;
     --) break;;

setup/container.sh

@@ -13,13 +13,10 @@ readonly DATA_DIR="/home/yellowtent/data"
 rm -rf "${CONFIG_DIR}"
 sudo -u yellowtent mkdir "${CONFIG_DIR}"
 
-########## logrotate (default ubuntu runs this daily)
-rm -rf /etc/logrotate.d/*
-cp -r "${container_files}/logrotate/." /etc/logrotate.d/
-
-########## supervisor
-rm -rf /etc/supervisor/*
-cp -r "${container_files}/supervisor/." /etc/supervisor/
+########## systemd
+cp -r "${container_files}/systemd/." /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable cloudron.target
 
 ########## sudoers
 rm /etc/sudoers.d/*
@@ -29,6 +26,10 @@ cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
 rm -rf /etc/collectd
 ln -sfF "${DATA_DIR}/collectd" /etc/collectd
 
+########## apparmor docker profile
+cp "${container_files}/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
+systemctl restart apparmor
+
 ########## nginx
 # link nginx config to system config
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx

setup/container/docker-cloudron-app.apparmor

@@ -0,0 +1,32 @@
+#include <tunables/global>
+
+
+profile docker-cloudron-app flags=(attach_disconnected,mediate_deleted) {
+
+  #include <abstractions/base>
+
+  ptrace peer=@{profile_name},
+
+  network,
+  capability,
+  file,
+  umount,
+
+  deny @{PROC}/sys/fs/** wklx,
+  deny @{PROC}/sysrq-trigger rwklx,
+  deny @{PROC}/mem rwklx,
+  deny @{PROC}/kmem rwklx,
+  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
+  deny @{PROC}/sys/kernel/*/** wklx,
+
+  deny mount,
+
+  deny /sys/[^f]*/** wklx,
+  deny /sys/f[^s]*/** wklx,
+  deny /sys/fs/[^c]*/** wklx,
+  deny /sys/fs/c[^g]*/** wklx,
+  deny /sys/fs/cg[^r]*/** wklx,
+  deny /sys/firmware/efi/efivars/** rwklx,
+  deny /sys/kernel/security/** rwklx,
+}
+

setup/container/logrotate/cloudron

@@ -1,6 +0,0 @@
-/var/log/cloudron/*log {
-    missingok
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/logrotate/supervisor

@@ -1,7 +0,0 @@
-/var/log/supervisor/*log {
-    missingok
-    copytruncate
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/supervisor/conf.d/apphealthtask.conf

@@ -1,10 +0,0 @@
-[program:apphealthtask]
-command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/box.conf

@@ -1,10 +0,0 @@
-[program:box]
-command=/usr/bin/node "/home/yellowtent/box/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/crashnotifier.conf

@@ -1,11 +0,0 @@
-[eventlistener:crashnotifier]
-command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
-events=PROCESS_STATE
-autostart=true
-autorestart=true
-redirect_stderr=false
-stderr_logfile=/var/log/supervisor/crashnotifier.log
-stderr_logfile_maxbytes=50MB
-stderr_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/janitor.conf

@@ -1,10 +0,0 @@
-[program:janitor]
-command=/usr/bin/node "/home/yellowtent/box/janitor.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/janitor.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/oauthproxy.conf

@@ -1,10 +0,0 @@
-[program:oauthproxy]
-command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/oauthproxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/supervisord.conf

@@ -1,33 +0,0 @@
-; supervisor config file
-
-; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[supervisord]
-logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
-pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-nodaemon = false
-childlogdir = /var/log/supervisor/
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=http://127.0.0.1:9001
-
-; The [include] section can just contain the "files" setting.  This
-; setting can list multiple files (separated by whitespace or
-; newlines).  It can also contain wildcards.  The filenames are
-; interpreted as relative to this file.  Included files *cannot*
-; include files themselves.
-
-[include]
-files = conf.d/*.conf
-

setup/container/systemd/box.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smart Cloud
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service janitor.timer
+After=box.service janitor.timer
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/container/systemd/janitor.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Cloudron Janitor
+OnFailure=crashnotifier@%n.service
+
+[Service]
+Type=simple
+WorkingDirectory=/home/yellowtent/box
+Restart=no
+ExecStart=/usr/bin/node /home/yellowtent/box/janitor.js
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M
+WatchdogSec=30

setup/container/systemd/janitor.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Janitor
+StopWhenUnneeded=true
+
+[Timer]
+# this activates it immediately
+OnBootSec=0
+OnUnitActiveSec=30min
+Unit=janitor.service
+

setup/start.sh

@@ -122,6 +122,7 @@ set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
 echo "Creating cloudron.conf"
+# note that arg_aws is a javascript object and intentionally unquoted below
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
@@ -138,7 +139,9 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "password": "${mysql_root_password}",
         "port": 3306,
         "name": "box"
-    }
+    },
+    "backupKey": "${arg_backup_key}",
+    "aws": ${arg_aws}
 }
 CONF_END
 
@@ -163,22 +166,10 @@ ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
 mysql -u root -p${mysql_root_password} \
     -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "80" "Reloading supervisor"
-# looks like restarting supervisor completely is the only way to reload it
-service supervisor stop || true
+set_progress "80" "Starting Cloudron"
+systemctl start cloudron.target
 
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
+sleep 2 # give systemd sometime to start the processes
 
 set_progress "85" "Reloading nginx"
 nginx -s reload

setup/start/collectd.conf

@@ -220,7 +220,7 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin processes>
-   ProcessMatch "app" "node app.js"
+   ProcessMatch "app" "node box.js"
 </Plugin>
 
 <Plugin swap>

setup/start/nginx/appconfig.ejs

@@ -69,7 +69,7 @@ server {
         }
 
 <% } else if ( endpoint === 'oauthproxy' ) { %>
-        proxy_pass http://127.0.0.1:4000;
+        proxy_pass http://127.0.0.1:3003;
         proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
 <% } else if ( endpoint === 'app' ) { %>
         proxy_pass http://127.0.0.1:<%= port %>;

setup/start/setup_infra.sh

@@ -28,6 +28,8 @@ fi
 
 # graphite
 graphite_container_id=$(docker run --restart=always -d --name="graphite" \
+    -m 75m \
+    --memory-swap 150m \
     -p 127.0.0.1:2003:2003 \
     -p 127.0.0.1:2004:2004 \
     -p 127.0.0.1:8000:8000 \
@@ -37,6 +39,8 @@ echo "Graphite container id: ${graphite_container_id}"
 
 # mail
 mail_container_id=$(docker run --restart=always -d --name="mail" \
+    -m 75m \
+    --memory-swap 150m \
     -p 127.0.0.1:25:25 \
     -h "${arg_fqdn}" \
     -e "DOMAIN_NAME=${arg_fqdn}" \
@@ -52,6 +56,8 @@ readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
 readonly MYSQL_ROOT_HOST='${docker0_ip}'
 EOF
 mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mysql:/var/lib/mysql" \
     -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
@@ -64,6 +70,8 @@ cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
 readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
 EOF
 postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
     -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
@@ -76,6 +84,8 @@ cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
 readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
 EOF
 mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
     -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \

setup/stop.sh

@@ -2,14 +2,6 @@
 
 set -eu -o pipefail
 
-echo "Stopping box code"
-
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
+echo "Stopping cloudron"
 
+systemctl stop cloudron.target

src/addons.js

@@ -38,8 +38,7 @@ var appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
     util = require('util'),
     uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
-    _ = require('underscore');
+    vbox = require('./vbox.js');
 
 var NOOP = function (app, callback) { return callback(); };
 
@@ -665,7 +664,7 @@ function setupRedis(app, callback) {
         name: 'redis-' + app.id,
         Hostname: config.appFqdn(app.location),
         Tty: true,
-        Image: 'cloudron/redis:0.3.1',
+        Image: 'cloudron/redis:0.3.2', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
         Volumes: {},
         VolumesFrom: []
@@ -678,6 +677,8 @@ function setupRedis(app, callback) {
             redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
             redisDataDir + ':/var/lib/redis:rw'
         ],
+        Memory: 1024 * 1024 * 75, // 100mb
+        MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
         // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work
         // On linux, export to localhost only for testing purposes and not for the app itself
         PortBindings: {

src/appdb.js

@@ -6,6 +6,7 @@ exports = module.exports = {
     get: get,
     getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
+    getByContainerId: getByContainerId,
     add: add,
     exists: exists,
     del: del,
@@ -35,12 +36,13 @@ exports = module.exports = {
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
-    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
     RSTATE_STOPPED: 'stopped', // app stopped by use
+    RSTATE_ERROR: 'error',
 
+    // run codes (keep in sync in UI)
     HEALTH_HEALTHY: 'healthy',
     HEALTH_UNHEALTHY: 'unhealthy',
     HEALTH_ERROR: 'error',
@@ -144,6 +146,22 @@ function getByHttpPort(httpPort, callback) {
     });
 }
 
+function getByContainerId(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -335,6 +353,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
 
     // Rules are:
     // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
     // update and configure are allowed only in installed state
 

src/apphealthmonitor.js

@@ -1,44 +1,33 @@
-#!/usr/bin/env node
-
 'use strict';
 
-require('supererror')({ splatchError: true });
-
-var appdb = require('./src/appdb.js'),
+var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    database = require('./src/database.js'),
-    DatabaseError = require('./src/databaseerror.js'),
-    debug = require('debug')('box:apphealthtask'),
-    docker = require('./src/docker.js'),
-    mailer = require('./src/mailer.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apphealthmonitor'),
+    docker = require('./docker.js'),
+    mailer = require('./mailer.js'),
     superagent = require('superagent'),
     util = require('util');
 
 exports = module.exports = {
-    run: run
+    start: start,
+    stop: stop
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
 var gHealthInfo = { }; // { time, emailSent }
+var gRunTimeout = null;
+var gDockerEventStream = null;
 
-function debugApp(app, args) {
+function debugApp(app) {
     assert(!app || typeof app === 'object');
 
     var prefix = app ? app.location : '(no app)';
     debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.initialize,
-        mailer.initialize
-    ], callback);
-}
-
 function setHealth(app, health, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof health, 'string');
@@ -130,18 +119,68 @@ function run() {
     processApps(function (error) {
         if (error) console.error(error);
 
-        setTimeout(run, HEALTHCHECK_INTERVAL);
+        gRunTimeout = setTimeout(run, HEALTHCHECK_INTERVAL);
     });
 }
 
-if (require.main === module) {
-    initialize(function (error) {
-        if (error) {
-            console.error('apphealth task exiting with error', error);
-            process.exit(1);
-        }
+/*
+    OOM can be tested using stress tool like so:
+        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        apt-get update && apt-get install stress
+        stress --vm 1 --vm-bytes 200M --vm-hang 0
+*/
+function processDockerEvents() {
+    // note that for some reason, the callback is called only on the first event
+    debug('Listening for docker events');
+    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return console.error(error);
 
-        run();
+        gDockerEventStream = stream;
+
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            var ev = JSON.parse(data);
+            debug('Container ' + ev.id + ' went OOM');
+            appdb.getByContainerId(ev.id, function (error, app) {
+                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
+                var context = JSON.stringify(ev);
+                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
+
+                debug('OOM Context: %s', context);
+                mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+            });
+        });
+
+        stream.on('error', function (error) {
+            console.error('Error reading docker events', error);
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+
+        stream.on('end', function () {
+            console.error('Docke event stream ended');
+            gDockerEventStream = null; // TODO: reconnect?
+            stream.end();
+        });
     });
 }
 
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting apphealthmonitor');
+
+    processDockerEvents();
+
+    run();
+
+    callback();
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    clearTimeout(gRunTimeout);
+    gDockerEventStream.end();
+
+    callback();
+}

src/apps.js

@@ -140,16 +140,18 @@ function validatePortBindings(portBindings, tcpPorts) {
     // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
     // for custom tcp ports
     var RESERVED_PORTS = [
-        22, /* ssh */
         25, /* smtp */
         53, /* dns */
         80, /* http */
         443, /* https */
+        919, /* ssh */
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
         2020, /* install server */
         config.get('port'), /* app server (lo) */
         config.get('internalPort'), /* internal app server (lo) */
+        config.get('ldapPort'), /* ldap server (lo) */
+        config.get('oauthProxyPort'), /* oauth proxy server (lo) */
         3306, /* mysql (lo) */
         8000 /* graphite (lo) */
     ];
@@ -490,28 +492,30 @@ function restore(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var restoreConfig = app.lastBackupConfig;
-        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+        // restore without a backup is the same as re-install
+        var restoreConfig = app.lastBackupConfig, values = { };
+        if (restoreConfig) {
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
-        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
-        error = checkManifestConstraints(restoreConfig.manifest);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
+            if (error) return callback(error);
 
-        error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
-        if (error) return callback(error);
+            // ## should probably query new location, access restriction from user
+            values = {
+                manifest: restoreConfig.manifest,
+                portBindings: restoreConfig.portBindings,
 
-        // ## should probably query new location, access restriction from user
-        var values = {
-            manifest: restoreConfig.manifest,
-            portBindings: restoreConfig.portBindings,
-
-            oldConfig: {
-                location: app.location,
-                accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings,
-                manifest: app.manifest
-            }
-        };
+                oldConfig: {
+                    location: app.location,
+                    accessRestriction: app.accessRestriction,
+                    portBindings: app.portBindings,
+                    manifest: app.manifest
+                }
+            };
+        }
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -573,6 +577,8 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
         return new Error('Box version exceeds Apps maxBoxVersion');
     }
@@ -645,27 +651,38 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     assert.strictEqual(typeof callback, 'function');
 
     function canAutoupdateApp(app, newManifest) {
-        // TODO: maybe check the description as well?
-        if (!newManifest.tcpPorts && !app.portBindings) return true;
-        if (!newManifest.tcpPorts || !app.portBindings) return false;
+        var tcpPorts = newManifest.tcpPorts || { };
+        var portBindings = app.portBindings; // this is never null
 
-        for (var env in newManifest.tcpPorts) {
-            if (!(env in app.portBindings)) return false;
-       }
+        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
+        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
+        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
 
-        return true;
+        for (var env in tcpPorts) {
+            if (!(env in portBindings)) return new Error(env + ' is required from user');
+        }
+
+        // it's fine if one or more keys got removed
+        return null;
     }
 
     if (!updateInfo) return callback(null);
 
     async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
         get(appId, function (error, app) {
-            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+            if (error) {
+                debug('Cannot autoupdate app %s : %s', appId, error.message);
+                return iteratorDone();
+           }
+
+            error = canAutoupdateApp(app, updateInfo[appId].manifest);
+            if (error) {
+                debug('app %s requires manual update. %s', appId, error.message);
                 return iteratorDone();
             }
 
-           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
-                if (error) debug('Error initiating autoupdate of %s', appId);
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+                if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);
 
                 iteratorDone(null);
             });
@@ -673,34 +690,36 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     }, callback);
 }
 
-function backupApp(app, addonsToBackup, callback) {
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldBackup(app, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    backups.copyLastBackup(app, function (error, newBackupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewBackup(app, addonsToBackup, callback) {
     assert.strictEqual(typeof app, 'object');
     assert(!addonsToBackup || typeof addonsToBackup, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    function canBackupApp(app) {
-        // only backup apps that are installed or pending configure. Rest of them are in some
-        // state not good for consistent backup (i.e addons may not have been setup completely)
-        return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
-                app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
-                app.installationState === appdb.ISTATE_PENDING_BACKUP ||
-                app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
-    }
-
-    if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy'));
-
-    var appConfig = {
-        manifest: app.manifest,
-        location: app.location,
-        portBindings: app.portBindings,
-        accessRestriction: app.accessRestriction
-    };
-
-    if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
-        return callback(safe.error);
-    }
-
-    backups.getBackupUrl(app, null, function (error, result) {
+    backups.getBackupUrl(app, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -709,18 +728,54 @@ function backupApp(app, addonsToBackup, callback) {
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
             addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            debugApp(app, 'backupApp: successful id:%s', result.id);
+            callback(null, result.id);
+        });
+    });
+}
 
-            setRestorePoint(app.id, result.id, appConfig, function (error) {
-                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+function backupApp(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-                return callback(null, result.id);
-            });
+    var appConfig = null, backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        appConfig = app.lastBackupConfig;
+        backupFunction = reuseOldBackup.bind(null, app);
+    } else {
+        appConfig = {
+            manifest: app.manifest,
+            location: app.location,
+            portBindings: app.portBindings,
+            accessRestriction: app.accessRestriction
+        };
+        backupFunction = createNewBackup.bind(null, app, addonsToBackup);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, appConfig, function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            return callback(null, backupId);
         });
     });
 }
@@ -756,7 +811,7 @@ function restoreApp(app, addonsToRestore, callback) {
 
         debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
 
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             addons.restoreAddons(app, addonsToRestore, callback);

src/apptask.js

@@ -25,6 +25,12 @@ exports = module.exports = {
 
 require('supererror')({ splatchError: true });
 
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
 var addons = require('./addons.js'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
@@ -46,6 +52,8 @@ var addons = require('./addons.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    SubdomainError = require('./subdomainerror.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
@@ -248,7 +256,7 @@ function deleteImage(app, manifest, callback) {
             noprune: false
         };
 
-        // delete image by id because docker pull pulls down all the tags and this is the only way to delete all tags
+         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
         docker.getImage(result.Id).remove(removeOptions, function (error) {
             if (error && error.statusCode === 404) return callback(null);
             if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -331,8 +339,12 @@ function startContainer(app, callback) {
             vbox.forwardFromHostToVirtualBox(app.id + '-tcp' + containerPort, hostPort);
         }
 
+        var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+
         var startOptions = {
             Binds: addons.getBindsSync(app, app.manifest.addons),
+            Memory: memoryLimit / 2,
+            MemorySwap: memoryLimit, // Memory + Swap
             PortBindings: dockerPortBindings,
             PublishAllPorts: false,
             Links: addons.getLinksSync(app, app.manifest.addons),
@@ -340,7 +352,8 @@ function startContainer(app, callback) {
                 "Name": "always",
                 "MaximumRetryCount": 0
             },
-            CpuShares: 512 // relative to 1024 for system processes
+            CpuShares: 512, // relative to 1024 for system processes
+            SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
         };
 
         var container = docker.getContainer(app.containerId);
@@ -355,6 +368,11 @@ function startContainer(app, callback) {
 }
 
 function stopContainer(app, callback) {
+    if (!app.containerId) {
+        debugApp(app, 'No previous container to stop');
+        return callback();
+    }
+
     var container = docker.getContainer(app.containerId);
     debugApp(app, 'Stopping container %s', container.id);
 
@@ -414,49 +432,47 @@ function downloadIcon(app, callback) {
 }
 
 function registerSubdomain(app, callback) {
-    debugApp(app, 'Registering subdomain location [%s]', app.location);
-
     // even though the bare domain is already registered in the appstore, we still
     // need to register it so that we have a dnsRecordId to wait for it to complete
     var record = { subdomain: app.location, type: 'A', value: sysinfo.getIp() };
 
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: [ record ] })
-        .end(function (error, res) {
-            if (error) return callback(error);
+    async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Registering subdomain location [%s]', app.location);
 
-            debugApp(app, 'Registered subdomain status: %s', res.status);
+        subdomains.add(record, function (error, changeId) {
+            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
 
-            if (res.status === 409) return callback(null); // already registered
-            if (res.status !== 201) return callback(new Error(util.format('Subdomain Registration failed. %s %j', res.status, res.body)));
-
-            updateApp(app, { dnsRecordId: res.body.ids[0] }, callback);
+            retryCallback(null, error || changeId);
         });
+    }, function (error, result) {
+        if (error || result instanceof Error) return callback(error || result);
+
+        updateApp(app, { dnsRecordId: result }, callback);
+    });
 }
 
-function unregisterSubdomain(app, callback) {
-    debugApp(app, 'Unregistering subdomain: dnsRecordId=%s', app.dnsRecordId);
-
-    if (!app.dnsRecordId) return callback(null);
-
+function unregisterSubdomain(app, location, callback) {
     // do not unregister bare domain because we show a error/cloudron info page there
-    if (app.location === '') return updateApp(app, { dnsRecordId: null }, callback);
+    if (location === '') {
+        debugApp(app, 'Skip unregister of empty subdomain');
+        return callback(null);
+    }
 
-    superagent
-        .del(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId)
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) {
-                debugApp(app, 'Error making request: %s', error);
-            } else if (res.status !== 204) {
-                debugApp(app, 'Error unregistering subdomain:', res.status, res.body);
-            }
+    var record = { subdomain: location, type: 'A', value: sysinfo.getIp() };
 
-            updateApp(app, { dnsRecordId: null }, callback);
+    async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Unregistering subdomain: %s', location);
+
+        subdomains.remove(record, function (error) {
+            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR))return retryCallback(error); // try again
+
+            retryCallback(error);
         });
+    }, function (error) {
+        if (error) debugApp(app, 'Error unregistering subdomain: %s', error);
+
+        updateApp(app, { dnsRecordId: null }, callback);
+    });
 }
 
 function removeIcon(app, callback) {
@@ -477,21 +493,15 @@ function waitForDnsPropagation(app, callback) {
         setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
     }
 
-    superagent
-        .get(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId + '/status')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+    subdomains.status(app.dnsRecordId, function (error, result) {
+        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
 
-            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, res.status);
+        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-            if (res.status !== 200) return retry(new Error(util.format('Error getting record status: %s %j', res.status, res.body)));
+        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-            if (res.body.status !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, res.body.status)));
-
-            callback(null);
-        });
+        callback(null);
+    });
 }
 
 // updates the app object and the database
@@ -530,9 +540,9 @@ function install(app, callback) {
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
         deleteVolume.bind(null, app),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
         removeOAuthProxyCredentials.bind(null, app),
-        removeIcon.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
         unconfigureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
@@ -617,7 +627,11 @@ function restore(app, callback) {
          // oldConfig can be null during upgrades
         addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
         deleteVolume.bind(null, app),
-        deleteImage.bind(null, app, app.manifest),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -671,16 +685,15 @@ function restore(app, callback) {
 
 // note that configure is called after an infra update as well
 function configure(app, callback) {
-    var locationChanged = app.oldConfig ? app.oldConfig.location !== app.location : true;
-
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         function (next) {
-            if (!locationChanged) return next();
-            unregisterSubdomain(app, next);
+            // oldConfig can be null during an infra update. location can be null when infra updated for an updated app
+            if (!app.oldConfig || !app.oldConfig.location || app.oldConfig.location === app.location) return next();
+            unregisterSubdomain(app, app.oldConfig.location, next);
         },
         removeOAuthProxyCredentials.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -691,14 +704,8 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
         allocateOAuthProxyCredentials.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
-                registerSubdomain.bind(null, app)
-            ], next);
-        },
+        updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
+        registerSubdomain.bind(null, app),
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         updateApp.bind(null, app, { installationProgress: '50, Setting up addons' }),
@@ -712,14 +719,8 @@ function configure(app, callback) {
 
         runApp.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
-                exports._waitForDnsPropagation.bind(null, app)
-            ], next);
-        },
+        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
+        exports._waitForDnsPropagation.bind(null, app),
 
         // done!
         function (callback) {
@@ -753,7 +754,11 @@ function update(app, callback) {
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, unusedAddons),
-        deleteImage.bind(null, app, app.manifest), // delete image even if did not change (see df158b111f)
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
 
         function (next) {
@@ -819,7 +824,7 @@ function uninstall(app, callback) {
         deleteImage.bind(null, app, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
 
         updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
         removeOAuthProxyCredentials.bind(null, app),

src/aws.js

@@ -0,0 +1,282 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl,
+
+    addSubdomain: addSubdomain,
+    delSubdomain: delSubdomain,
+    getChangeStatus: getChangeStatus,
+
+    copyObject: copyObject
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('./config.js'),
+    debug = require('debug')('box:aws'),
+    SubdomainError = require('./subdomainerror.js'),
+    superagent = require('superagent');
+
+function getAWSCredentials(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
+        superagent.post(url).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 201) return callback(new Error(result.text));
+            if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
+
+            var credentials = {
+                accessKeyId: result.body.credentials.AccessKeyId,
+                secretAccessKey: result.body.credentials.SecretAccessKey,
+                sessionToken: result.body.credentials.SessionToken,
+                region: 'us-east-1'
+            };
+
+            if (config.aws().endpoint) credentials.endpoint = new AWS.Endpoint(config.aws().endpoint);
+
+            callback(null, credentials);
+        });
+    } else {
+        if (!config.aws().accessKeyId || !config.aws().secretAccessKey) return callback(new SubdomainError(SubdomainError.MISSING_CREDENTIALS));
+
+        var credentials = {
+            accessKeyId: config.aws().accessKeyId,
+            secretAccessKey: config.aws().secretAccessKey,
+            region: 'us-east-1'
+        };
+
+        if (config.aws().endpoint) credentials.endpoint = new AWS.Endpoint(config.aws().endpoint);
+
+        callback(null, credentials);
+    }
+}
+
+function getSignedUploadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedUploadUrl: %s', filename);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedDownloadUrl: %s', filename);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getZoneByName(zoneName, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getZoneByName: %s', zoneName);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.listHostedZones({}, function (error, result) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+
+            var zone = result.HostedZones.filter(function (zone) {
+                return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+            })[0];
+
+            if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
+
+            debug('getZoneByName: found zone', zone);
+
+            callback(null, zone);
+        });
+    });
+}
+
+function addSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addSubdomain: ' + subdomain + ' for domain ' + zoneName + ' with value ' + value);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'UPSERT',
+                    ResourceRecordSet: {
+                        Type: type,
+                        Name: fqdn,
+                        ResourceRecords: [{
+                            Value: value
+                        }],
+                        Weight: 0,
+                        SetIdentifier: fqdn,
+                        TTL: 1
+                    }
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.code === 'PriorRequestNotComplete') {
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+                } else if (error) {
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+                }
+
+                debug('addSubdomain: success. changeInfoId:%j', result);
+
+                callback(null, result.ChangeInfo.Id);
+            });
+        });
+    });
+}
+
+function delSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('delSubdomain: %s for domain %s.', subdomain, zoneName);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var resourceRecordSet = {
+            Name: fqdn,
+            Type: type,
+            ResourceRecords: [{
+                Value: value
+            }],
+            Weight: 0,
+            SetIdentifier: fqdn,
+            TTL: 1
+        };
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'DELETE',
+                    ResourceRecordSet: resourceRecordSet
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                    debug('delSubdomain: resource record set not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'NoSuchHostedZone') {
+                    debug('delSubdomain: hosted zone not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'PriorRequestNotComplete') {
+                    debug('delSubdomain: resource is still busy', error);
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                } else if (error && error.code === 'InvalidChangeBatch') {
+                    debug('delSubdomain: invalid change batch. No such record to be deleted.');
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error) {
+                    debug('delSubdomain: error', error);
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                }
+
+                debug('delSubdomain: success');
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function getChangeStatus(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.getChange({ Id: changeId }, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, result.ChangeInfo.Status);
+        });
+    });
+}
+
+function copyObject(from, to, callback) {
+    assert.strictEqual(typeof from, 'string');
+    assert.strictEqual(typeof to, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var params = {
+            Bucket: config.aws().backupBucket, // target bucket
+            Key: config.aws().backupPrefix + '/' + to, // target file
+            CopySource: config.aws().backupBucket + '/' + config.aws().backupPrefix + '/' + from, // source
+        };
+
+        var s3 = new AWS.S3(credentials);
+        s3.copyObject(params, callback);
+    });
+}

src/backups.js

@@ -6,10 +6,13 @@ exports = module.exports = {
     getAllPaged: getAllPaged,
 
     getBackupUrl: getBackupUrl,
-    getRestoreUrl: getRestoreUrl
+    getRestoreUrl: getRestoreUrl,
+
+    copyLastBackup: copyLastBackup
 };
 
 var assert = require('assert'),
+    aws = require('./aws.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
     superagent = require('superagent'),
@@ -54,42 +57,63 @@ function getAllPaged(page, perPage, callback) {
     });
 }
 
-function getBackupUrl(app, appBackupIds, callback) {
+function getBackupUrl(app, callback) {
     assert(!app || typeof app === 'object');
-    assert(!appBackupIds || util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+    var filename = '';
+    if (app) {
+        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    } else {
+        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
+    }
 
-    var data = {
-        boxVersion: config.version(),
-        appId: app ? app.id : null,
-        appVersion: app ? app.manifest.version : null,
-        appBackupIds: appBackupIds
-    };
+    aws.getSignedUploadUrl(filename, function (error, result) {
+        if (error) return callback(error);
 
-    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        var obj = {
+            id: filename,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+        callback(null, obj);
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+    aws.getSignedDownloadUrl(backupId, function (error, result) {
+        if (error) return callback(error);
 
-    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        var obj = {
+            id: backupId,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+        callback(null, obj);
     });
 }
 
+function copyLastBackup(app, callback) {
+    assert(app && typeof app === 'object');
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
+    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    aws.copyObject(app.lastBackupId, toFilename, function (error) {
+        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+        return callback(null, toFilename);
+    });
+}

src/caas.js

@@ -0,0 +1,88 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    addSubdomain: addSubdomain,
+    delSubdomain: delSubdomain,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:caas'),
+    SubdomainError = require('./subdomainerror.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function addSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addSubdomain: ' + subdomain + ' for domain ' + zoneName + ' with value ' + value);
+
+    var data = {
+        type: type,
+        value: value
+    };
+
+    superagent
+        .post(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
+        .query({ token: config.token() })
+        .send(data)
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.status !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.changeId);
+        });
+}
+
+function delSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('delSubdomain: %s for domain %s.', subdomain, zoneName);
+
+    var data = {
+        type: type,
+        value: value
+    };
+
+    superagent
+        .del(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
+        .query({ token: config.token() })
+        .send(data)
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.status !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null);
+        });
+}
+
+function getChangeStatus(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
+        .query({ token: config.token() })
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.status);
+        });
+
+}

src/cloudron.js

@@ -39,6 +39,7 @@ var apps = require('./apps.js'),
     settings = require('./settings.js'),
     SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
@@ -46,7 +47,8 @@ var apps = require('./apps.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
 
 var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
@@ -54,7 +56,7 @@ var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
 
-var gAddMailDnsRecordsTimerId = null,
+var gAddDnsRecordsTimerId = null,
     gCloudronDetails = null;            // cached cloudron details like region,size...
 
 function debugApp(app, args) {
@@ -108,20 +110,17 @@ function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     if (process.env.BOX_ENV !== 'test') {
-        addMailDnsRecords();
+        addDnsRecords();
     }
 
-    // Send heartbeat once we are up and running, this speeds up the Cloudron creation, as otherwise we are bound to the cron.js settings
-    sendHeartbeat();
-
     callback(null);
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clearTimeout(gAddMailDnsRecordsTimerId);
-    gAddMailDnsRecordsTimerId = null;
+    clearTimeout(gAddDnsRecordsTimerId);
+    gAddDnsRecordsTimerId = null;
 
     callback(null);
 }
@@ -139,7 +138,7 @@ function setTimeZone(ip, callback) {
         }
 
         if (!result.body.timezone) {
-            debug('No timezone in geoip response');
+            debug('No timezone in geoip response : %j', result.body);
             return callback(null);
         }
 
@@ -159,7 +158,7 @@ function activate(username, password, email, name, ip, callback) {
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { });
+    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
     if (!name) name = settings.getDefaultSync(settings.CLOUDRON_NAME_KEY);
 
@@ -269,18 +268,20 @@ function getConfig(callback) {
 }
 
 function sendHeartbeat() {
+    // Only send heartbeats after the admin dns record is synced to give appstore a chance to know that fact
+    if (!config.get('dnsInSync')) return;
+
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
 
-    // TODO: this must be a POST
-    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
         if (error) debug('Error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
         else debug('Heartbeat sent to %s', url);
     });
 }
 
-function sendMailDnsRecordsRequest(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function addDnsRecords() {
+    if (config.get('dnsInSync')) return sendHeartbeat(); // already registered send heartbeat
 
     var DKIM_SELECTOR = 'mail';
     var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
@@ -288,13 +289,20 @@ function sendMailDnsRecordsRequest(callback) {
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
 
-    if (publicKey === null) return callback(new Error('Error reading dkim public key'));
+    if (publicKey === null) {
+        console.error('Error reading dkim public key. Stop DNS setup.');
+        return;
+    }
 
     // remove header, footer and new lines
     publicKey = publicKey.split('\n').slice(1, -2).join('');
 
     // note that dmarc requires special DNS records for external RUF and RUA
     var records = [
+        // naked domain
+        { subdomain: '', type: 'A', value: sysinfo.getIp() },
+        // webadmin domain
+        { subdomain: 'my', type: 'A', value: sysinfo.getIp() },
         // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
         { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
         // t=s limits the domainkey to this domain and not it's subdomains
@@ -303,38 +311,47 @@ function sendMailDnsRecordsRequest(callback) {
         { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
     ];
 
-    debug('sendMailDnsRecords request:%s', JSON.stringify(records));
+    debug('addDnsRecords:', records);
 
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: records })
-        .end(function (error, res) {
-            if (error) return callback(error);
-
-            debug('sendMailDnsRecords status: %s', res.status);
-
-            if (res.status === 409) return callback(null); // already registered
-
-            if (res.status !== 201) return callback(new Error(util.format('Failed to add Mail DNS records: %s %j', res.status, res.body)));
-
-            return callback(null, res.body.ids);
-        });
-}
-
-function addMailDnsRecords() {
-    if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
-
-    sendMailDnsRecordsRequest(function (error, ids) {
+    subdomains.addMany(records, function (error, changeIds) {
         if (error) {
-            console.error('Mail DNS record addition failed', error);
-            gAddMailDnsRecordsTimerId = setTimeout(addMailDnsRecords, 30000);
+            console.error('Admin DNS record addition failed', error);
+            gAddDnsRecordsTimerId = setTimeout(addDnsRecords, 10000);
             return;
         }
 
-        debug('Added Mail DNS records successfully');
-        config.set('mailDnsRecordIds', ids);
+        function checkIfInSync() {
+            debug('addDnsRecords: Check if admin DNS record is in sync.');
+
+            var allDone = true;
+
+            async.each(changeIds, function (changeId, callback) {
+                subdomains.status(changeId, function (error, result) {
+                    if (error) return callback(new Error('Failed to check if admin DNS record is in sync.', error));
+
+                    if (result !== 'done') allDone = false;
+
+                    callback(null);
+                });
+            }, function (error) {
+                if (error) console.error(error);
+
+                // retry if needed
+                if (error || !allDone) {
+                    gAddDnsRecordsTimerId = setTimeout(checkIfInSync, 5000);
+                    return;
+                }
+
+                config.set('dnsInSync', true);
+
+                // send heartbeat after the dns records are done
+                sendHeartbeat();
+
+                debug('addDnsRecords: done');
+            });
+        }
+
+        checkIfInSync();
     });
 }
 
@@ -492,17 +509,19 @@ function doUpdate(boxUpdateInfo, callback) {
             var args = {
                 sourceTarballUrl: result.body.url,
 
-                // IMPORTANT: if you change this, fix up argparser.sh as well. keep these sorted for readability
+                // this data is opaque to the installer
                 data: {
                     apiServerOrigin: config.apiServerOrigin(),
+                    aws: config.aws(),
+                    backupKey: config.backupKey(),
                     boxVersionsUrl: config.get('boxVersionsUrl'),
                     fqdn: config.fqdn(),
                     isCustomDomain: config.isCustomDomain(),
-                    restoreKey: null,
                     restoreUrl: null,
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    restoreKey: null,
                     token: config.token(),
+                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
                     version: boxUpdateInfo.version,
                     webServerOrigin: config.webServerOrigin()
                 }
@@ -530,6 +549,9 @@ function backup(callback) {
     var error = locker.lock(locker.OP_FULL_BACKUP);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    // clearing backup ensures tools can 'wait' on progress
+    progress.clear(progress.BACKUP);
+
     // start the backup operation in the background
     backupBoxAndApps(function (error) {
         if (error) console.error('backup failed.', error);
@@ -561,7 +583,7 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+    backups.getBackupUrl(null /* app */, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -569,14 +591,17 @@ function backupBoxWithAppBackupIds(appBackupIds, callback) {
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             debug('backup: successful');
 
-            callback(null, result.id);
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
+                callback(null, result.id);
+            });
         });
     });
 }
@@ -609,14 +634,14 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, 'Backing up app at ' + app.location);
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
 
-                if (error && error.reason === AppsError.BAD_STATE) {
-                    debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);
-                    backupId = app.lastBackupId;
+                if (error && error.reason !== AppsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
                 }
 
-                return iteratorCallback(null, backupId);
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
             });
         }, function appsBackedUp(error, backupIds) {
             if (error) {
@@ -624,7 +649,7 @@ function backupBoxAndApps(callback) {
                 return callback(error);
             }
 
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
 
             backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
                 progress.set(progress.BACKUP, 100, error ? error.message : '');

src/config.js

@@ -25,11 +25,15 @@ exports = module.exports = {
 
     // these values are derived
     adminOrigin: adminOrigin,
+    internalAdminOrigin: internalAdminOrigin,
     appFqdn: appFqdn,
     zoneName: zoneName,
 
     isDev: isDev,
 
+    backupKey: backupKey,
+    aws: aws,
+
     // for testing resets to defaults
     _reset: initConfig
 };
@@ -70,6 +74,15 @@ function initConfig() {
     data.webServerOrigin = null;
     data.internalPort = 3001;
     data.ldapPort = 3002;
+    data.oauthProxyPort = 3003;
+    data.backupKey = 'backupKey';
+    data.aws = {
+        backupBucket: null,
+        backupPrefix: null,
+        accessKeyId: null,      // selfhosting only
+        secretAccessKey: null   // selfhosting only
+    };
+    data.dnsInSync = false;
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -86,6 +99,9 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
+        data.aws.backupBucket = 'testbucket';
+        data.aws.backupPrefix = 'testprefix';
+        data.aws.endpoint = 'http://localhost:5353';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -99,6 +115,9 @@ function initConfig() {
     saveSync();
 }
 
+// cleanup any old config file we have for tests
+if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
+
 initConfig();
 
 // set(obj) or set(key, value)
@@ -145,6 +164,10 @@ function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
 
+function internalAdminOrigin() {
+    return 'http://127.0.0.1:' + get('port');
+}
+
 function token() {
     return get('token');
 }
@@ -172,3 +195,10 @@ function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
 
+function backupKey() {
+    return get('backupKey');
+}
+
+function aws() {
+    return get('aws');
+}

src/cron.js

@@ -48,6 +48,8 @@ function recreateJobs(unusedTimeZone, callback) {
     if (typeof unusedTimeZone === 'function') callback = unusedTimeZone;
 
     settings.getAll(function (error, allSettings) {
+        debug('Creating jobs with timezone %s', allSettings[settings.TIME_ZONE_KEY]);
+
         if (gHeartbeatJob) gHeartbeatJob.stop();
         gHeartbeatJob = new CronJob({
             cronTime: '00 */1 * * * *', // every minute
@@ -110,7 +112,7 @@ function autoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gBoxUpdateCheckerJob.cronTime.timeZone // hack
+        timeZone: gBoxUpdateCheckerJob.cronTime.zone // hack
     });
 }
 

src/ldap.js

@@ -1,7 +1,8 @@
 'use strict';
 
 exports = module.exports = {
-    start: start
+    start: start,
+    stop: stop
 };
 
 var assert = require('assert'),
@@ -25,10 +26,10 @@ var gLogger = {
 };
 
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
-var GROUP_ADMINS_DN = 'cn=admin,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
 function start(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     gServer = ldap.createServer({ log: gLogger });
 
@@ -123,3 +124,11 @@ function start(callback) {
 
     gServer.listen(config.get('ldapPort'), callback);
 }
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gServer.close();
+
+    callback();
+}

src/locker.js

@@ -9,6 +9,7 @@ function Locker() {
     this._operation = null;
     this._timestamp = null;
     this._watcherId = -1;
+    this._lockDepth = 0; // recursive locks
 }
 util.inherits(Locker, EventEmitter);
 
@@ -24,6 +25,7 @@ Locker.prototype.lock = function (operation) {
     if (this._operation !== null) return new Error('Already locked for ' + this._operation);
 
     this._operation = operation;
+    ++this._lockDepth;
     this._timestamp = new Date();
     var that = this;
     this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
@@ -35,17 +37,31 @@ Locker.prototype.lock = function (operation) {
     return null;
 };
 
+Locker.prototype.recursiveLock = function (operation) {
+    if (this._operation === operation) {
+        ++this._lockDepth;
+        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
+        return null;
+    }
+
+    return this.lock(operation);
+};
+
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
-    debug('Released : %s', this._operation);
+    if (--this._lockDepth === 0) {
+        debug('Released : %s', this._operation);
 
-    this._operation = null;
-    this._timestamp = null;
-    clearInterval(this._watcherId);
-    this._watcherId = -1;
+        this._operation = null;
+        this._timestamp = null;
+        clearInterval(this._watcherId);
+        this._watcherId = -1;
+    } else {
+        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
+    }
 
     this.emit('unlocked', operation);
 

src/mail_templates/crash_notification.ejs

@@ -2,7 +2,7 @@
 
 Dear Cloudron Team,
 
-unfortunately the <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
 
 Please see some excerpt of the logs below.
 
@@ -11,7 +11,7 @@ Your Cloudron
 
 -------------------------------------
 
-<%= context %>
+<%- context %>
 
 <% } else { %>
 

src/oauthproxy.js

@@ -0,0 +1,192 @@
+'use strict';
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var appdb = require('./appdb.js'),
+    assert = require('assert'),
+    clientdb = require('./clientdb.js'),
+    config = require('./config.js'),
+    debug = require('debug')('box:proxy'),
+    express = require('express'),
+    http = require('http'),
+    proxy = require('proxy-middleware'),
+    session = require('cookie-session'),
+    superagent = require('superagent'),
+    url = require('url'),
+    uuid = require('node-uuid');
+
+var gSessions = {};
+var gProxyMiddlewareCache = {};
+var gHttpServer = null;
+
+var CALLBACK_URI = '/callback';
+
+function attachSessionData(req, res, next) {
+    assert.strictEqual(typeof req.session, 'object');
+
+    if (!req.session.id || !gSessions[req.session.id]) {
+        req.session.id = uuid.v4();
+        gSessions[req.session.id] = {};
+    }
+
+    // attach the session data to the requeset
+    req.sessionData = gSessions[req.session.id];
+
+    next();
+}
+
+function verifySession(req, res, next) {
+    assert.strictEqual(typeof req.sessionData, 'object');
+
+    if (!req.sessionData.accessToken) {
+        req.authenticated = false;
+        return next();
+    }
+
+    // use http admin origin so that it works with self-signed certs
+    superagent
+        .get(config.internalAdminOrigin() + '/api/v1/profile')
+        .query({ access_token: req.sessionData.accessToken})
+        .end(function (error, result) {
+        if (error) {
+            console.error(error);
+            req.authenticated = false;
+        } else if (result.statusCode !== 200) {
+            req.sessionData.accessToken = null;
+            req.authenticated = false;
+        } else {
+            req.authenticated = true;
+        }
+
+        next();
+    });
+}
+
+function authenticate(req, res, next) {
+    // proceed if we are authenticated
+    if (req.authenticated) return next();
+
+    if (req.path === CALLBACK_URI && req.sessionData.returnTo) {
+        // exchange auth code for an access token
+        var query = {
+            response_type: 'token',
+            client_id: req.sessionData.clientId
+        };
+
+        var data = {
+            grant_type: 'authorization_code',
+            code: req.query.code,
+            redirect_uri: req.sessionData.returnTo,
+            client_id: req.sessionData.clientId,
+            client_secret: req.sessionData.clientSecret
+        };
+
+        // use http admin origin so that it works with self-signed certs
+        superagent
+            .post(config.internalAdminOrigin() + '/api/v1/oauth/token')
+            .query(query).send(data)
+            .end(function (error, result) {
+            if (error) {
+                console.error(error);
+                return res.send(500, 'Unable to contact the oauth server.');
+            }
+            if (result.statusCode !== 200) {
+                console.error('Failed to exchange auth code for a token.', result.statusCode, result.body);
+                return res.send(500, 'Failed to exchange auth code for a token.');
+            }
+
+            req.sessionData.accessToken = result.body.access_token;
+
+            debug('user verified.');
+
+            // now redirect to the actual initially requested URL
+            res.redirect(req.sessionData.returnTo);
+        });
+    } else {
+        var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
+
+        if (!Number.isFinite(port)) {
+            console.error('Failed to parse nginx proxy header to get app port.');
+            return res.send(500, 'Routing error. No forwarded port.');
+        }
+
+        debug('begin verifying user for app on port %s.', port);
+
+        appdb.getByHttpPort(port, function (error, result) {
+            if (error) {
+                console.error('Unknown app.', error);
+                return res.send(500, 'Unknown app.');
+            }
+
+            clientdb.getByAppId('proxy-' + result.id, function (error, result) {
+                if (error) {
+                    console.error('Unkonwn OAuth client.', error);
+                    return res.send(500, 'Unknown OAuth client.');
+                }
+
+                req.sessionData.port = port;
+                req.sessionData.returnTo = result.redirectURI + req.path;
+                req.sessionData.clientId = result.id;
+                req.sessionData.clientSecret = result.clientSecret;
+
+                var callbackUrl = result.redirectURI + CALLBACK_URI;
+                var scope = 'profile,roleUser';
+                var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + result.id + '&redirect_uri=' + callbackUrl + '&scope=' + scope;
+
+                debug('begin OAuth flow for client %s.', result.name);
+
+                // begin the OAuth flow
+                res.redirect(oauthLogin);
+            });
+        });
+    }
+}
+
+function forwardRequestToApp(req, res, next) {
+    var port = req.sessionData.port;
+
+    debug('proxy request for port %s with path %s.', port, req.path);
+
+    var proxyMiddleware = gProxyMiddlewareCache[port];
+    if (!proxyMiddleware) {
+        console.log('Adding proxy middleware for port %d', port);
+
+        proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
+        gProxyMiddlewareCache[port] = proxyMiddleware;
+    }
+
+    proxyMiddleware(req, res, next);
+}
+
+function initializeServer() {
+    var app = express();
+    var httpServer = http.createServer(app);
+
+    httpServer.on('error', console.error);
+
+    app
+        .use(session({ keys: ['blue', 'cheese', 'is', 'something'] }))
+        .use(attachSessionData)
+        .use(verifySession)
+        .use(authenticate)
+        .use(forwardRequestToApp);
+
+    return httpServer;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gHttpServer = initializeServer();
+
+    gHttpServer.listen(config.get('oauthProxyPort'), callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gHttpServer.close(callback);
+}

src/routes/internal.js

@@ -7,6 +7,7 @@ exports = module.exports = {
 };
 
 var cloudron = require('../cloudron.js'),
+    CloudronError = require('../cloudron.js').CloudronError,
     debug = require('debug')('box:routes/internal'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess;
@@ -14,10 +15,12 @@ var cloudron = require('../cloudron.js'),
 function backup(req, res, next) {
     debug('trigger backup');
 
+    // note that cloudron.backup only waits for backup initiation and not for backup to complete
+    // backup progress can be checked up ny polling the progress api call
     cloudron.backup(function (error) {
-        if (error) debug('Internal route backup failed', error);
-    });
+        if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error) return next(new HttpError(500, error));
 
-    // we always succeed to trigger a backup
-    next(new HttpSuccess(202, {}));
+        next(new HttpSuccess(202, {}));
+    });
 }

src/routes/oauth2.js

@@ -171,6 +171,8 @@ function sendErrorPageOrRedirect(req, res, message) {
     }
 }
 
+// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
+// This usually happens when the OAuth client ID is wrong
 function sendError(req, res, message) {
     assert.strictEqual(typeof req, 'object');
     assert.strictEqual(typeof res, 'object');

src/routes/test/apps-test.js

@@ -22,6 +22,7 @@ var appdb = require('../../appdb.js'),
     hock = require('hock'),
     http = require('http'),
     https = require('https'),
+    js2xml = require('js2xmlparser'),
     net = require('net'),
     nock = require('nock'),
     os = require('os'),
@@ -31,7 +32,6 @@ var appdb = require('../../appdb.js'),
     safe = require('safetydance'),
     server = require('../../server.js'),
     settings = require('../../settings.js'),
-    sysinfo = require('../../sysinfo.js'),
     tokendb = require('../../tokendb.js'),
     url = require('url'),
     util = require('util'),
@@ -51,6 +51,21 @@ var USERNAME_1 = 'user', PASSWORD_1 = 'password', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;
 
+ var awsHostedZones = {
+     HostedZones: [{
+         Id: '/hostedzone/ZONEID',
+         Name: 'localhost.',
+         CallerReference: '305AFD59-9D73-4502-B020-F4E6F889CB30',
+         ResourceRecordSetCount: 2,
+         ChangeInfo: {
+             Id: '/change/CKRTFJA0ANHXB',
+             Status: 'INSYNC'
+         }
+     }],
+    IsTruncated: false,
+    MaxItems: '100'
+ };
+
 function startDockerProxy(interceptor, callback) {
     assert.strictEqual(typeof interceptor, 'function');
 
@@ -79,10 +94,12 @@ function startDockerProxy(interceptor, callback) {
 
 function setup(done) {
     async.series([
-        server.start.bind(server),
-
+        // first clear, then start server. otherwise, taskmanager spins up tasks for obsolete appIds
+        database.initialize,
         database._clear,
 
+        server.start.bind(server),
+
         function (callback) {
             var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
             var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
@@ -104,11 +121,11 @@ function setup(done) {
             });
         },
 
-        child_process.exec.bind(null, __dirname + '/start_addons.sh'),
-
         function (callback) {
-            callback(null);
+            console.log('Starting addons, this can take 10 seconds');
+            child_process.exec(__dirname + '/start_addons.sh', callback);
         },
+
         function (callback) {
             request.post(SERVER_URL + '/api/v1/users')
                    .query({ access_token: token })
@@ -129,11 +146,12 @@ function setup(done) {
 }
 
 function cleanup(done) {
+    // db is not cleaned up here since it's too late to call it after server.stop. if called before server.stop taskmanager apptasks are unhappy :/
     async.series([
-        database._clear,
-
         server.stop,
 
+        function (callback) { setTimeout(callback, 2000); }, // give taskmanager tasks couple of seconds to finish
+
         child_process.exec.bind(null, 'docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb')
     ], done);
 }
@@ -143,10 +161,19 @@ describe('App API', function () {
     var dockerProxy;
 
     before(function (done) {
-        dockerProxy = startDockerProxy(function interceptor() { return false; }, function () {
+        dockerProxy = startDockerProxy(function interceptor(req, res) {
+            if (req.method === 'DELETE' && req.url === '/images/c7ddfc8fb7cd8a14d4d70153a199ff0c6e9b709807aeec5a7b799d60618731d1?force=true&noprune=false') {
+                res.writeHead(200);
+                res.end();
+                return true;
+            }
+
+            return false;
+        }, function () {
             setup(done);
         });
     });
+
     after(function (done) {
         APP_ID = null;
         cleanup(function () {
@@ -473,7 +500,8 @@ describe('App API', function () {
 describe('App installation', function () {
     this.timeout(50000);
 
-    var hockInstance = hock.createHock({ throwOnUnmatched: false }), hockServer, dockerProxy;
+    var apiHockInstance = hock.createHock({ throwOnUnmatched: false }), apiHockServer, dockerProxy;
+    var awsHockInstance = hock.createHock({ throwOnUnmatched: false }), awsHockServer;
     var imageDeleted = false, imageCreated = false;
 
     before(function (done) {
@@ -500,28 +528,42 @@ describe('App installation', function () {
             setup,
 
             function (callback) {
-                hockInstance
+                apiHockInstance
                     .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
                     .replyWithFile(200, path.resolve(__dirname, '../../../webadmin/src/img/appicon_fallback.png'))
-                    .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP_LOCATION, type: 'A', value: sysinfo.getIp() } ] })
-                    .reply(201, { ids: [ 'dnsrecordid' ] }, { 'Content-Type': 'application/json' })
-                    .delete('/api/v1/subdomains/dnsrecordid?token=' + config.token())
-                    .reply(204, { }, { 'Content-Type': 'application/json' });
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .max(Infinity)
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } }, { 'Content-Type': 'application/json' });
 
                 var port = parseInt(url.parse(config.apiServerOrigin()).port, 10);
-                hockServer = http.createServer(hockInstance.handler).listen(port, callback);
+                apiHockServer = http.createServer(apiHockInstance.handler).listen(port, callback);
+            },
+
+            function (callback) {
+                awsHockInstance
+                    .get('/2013-04-01/hostedzone')
+                    .max(Infinity)
+                    .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }), { 'Content-Type': 'application/xml' })
+                    .filteringRequestBody(function (unusedBody) { return ''; }) // strip out body
+                    .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+                    .max(Infinity)
+                    .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'dnsrecordid', Status: 'INSYNC' } }), { 'Content-Type': 'application/xml' });
+
+                var port = parseInt(url.parse(config.aws().endpoint).port, 10);
+                awsHockServer = http.createServer(awsHockInstance.handler).listen(port, callback);
             }
         ], done);
     });
 
     after(function (done) {
         APP_ID = null;
-        cleanup(function (error) {
-            if (error) return done(error);
-            hockServer.close(function () {
-                dockerProxy.close(done);
-            });
-        });
+
+        async.series([
+            cleanup,
+            apiHockServer.close.bind(apiHockServer),
+            awsHockServer.close.bind(awsHockServer),
+            dockerProxy.close.bind(dockerProxy)
+        ], done);
     });
 
     var appResult = null /* the json response */, appEntry = null /* entry from database */;
@@ -882,9 +924,13 @@ describe('App installation', function () {
     });
 
     it('uninstalled - unregistered subdomain', function (done) {
-        hockInstance.done(function (error) { // checks if all the hockServer APIs were called
+        apiHockInstance.done(function (error) { // checks if all the apiHockServer APIs were called
             expect(!error).to.be.ok();
-            done();
+
+            awsHockInstance.done(function (error) {
+                expect(!error).to.be.ok();
+                done();
+            });
         });
     });
 
@@ -904,7 +950,8 @@ describe('App installation', function () {
 describe('App installation - port bindings', function () {
     this.timeout(50000);
 
-    var hockInstance = hock.createHock({ throwOnUnmatched: false }), hockServer, dockerProxy;
+    var apiHockInstance = hock.createHock({ throwOnUnmatched: false }), apiHockServer, dockerProxy;
+    var awsHockInstance = hock.createHock({ throwOnUnmatched: false }), awsHockServer;
     var imageDeleted = false, imageCreated = false;
 
     before(function (done) {
@@ -930,35 +977,41 @@ describe('App installation - port bindings', function () {
             setup,
 
             function (callback) {
-                hockInstance
-                    // app install
+                apiHockInstance
                     .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
                     .replyWithFile(200, path.resolve(__dirname, '../../../webadmin/src/img/appicon_fallback.png'))
-                    .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP_LOCATION, type: 'A', value: sysinfo.getIp() } ] })
-                    .reply(201, { ids: [ 'dnsrecordid' ] }, { 'Content-Type': 'application/json' })
-                    // app configure
-                    .delete('/api/v1/subdomains/dnsrecordid?token=' + config.token())
-                    .reply(204, { }, { 'Content-Type': 'application/json' })
-                    .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP_LOCATION_NEW, type: 'A', value: sysinfo.getIp() } ] })
-                    .reply(201, { ids: [ 'anotherdnsid' ] }, { 'Content-Type': 'application/json' })
-                    // app remove
-                    .delete('/api/v1/subdomains/anotherdnsid?token=' + config.token())
-                    .reply(204, { }, { 'Content-Type': 'application/json' });
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .max(Infinity)
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } }, { 'Content-Type': 'application/json' });
 
                 var port = parseInt(url.parse(config.apiServerOrigin()).port, 10);
-                hockServer = http.createServer(hockInstance.handler).listen(port, callback);
+                apiHockServer = http.createServer(apiHockInstance.handler).listen(port, callback);
+            },
+
+            function (callback) {
+                awsHockInstance
+                    .get('/2013-04-01/hostedzone')
+                    .max(Infinity)
+                    .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }), { 'Content-Type': 'application/xml' })
+                    .filteringRequestBody(function (unusedBody) { return ''; }) // strip out body
+                    .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+                    .max(Infinity)
+                    .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'dnsrecordid', Status: 'INSYNC' } }), { 'Content-Type': 'application/xml' });
+
+                var port = parseInt(url.parse(config.aws().endpoint).port, 10);
+                awsHockServer = http.createServer(awsHockInstance.handler).listen(port, callback);
             }
         ], done);
     });
 
     after(function (done) {
         APP_ID = null;
-        cleanup(function (error) {
-            if (error) return done(error);
-            hockServer.close(function () {
-                dockerProxy.close(done);
-            });
-        });
+        async.series([
+            cleanup,
+            apiHockServer.close.bind(apiHockServer),
+            awsHockServer.close.bind(awsHockServer),
+            dockerProxy.close.bind(dockerProxy)
+        ], done);
     });
 
     var appResult = null, appEntry = null;
@@ -1302,9 +1355,13 @@ describe('App installation - port bindings', function () {
     });
 
     it('uninstalled - unregistered subdomain', function (done) {
-        hockInstance.done(function (error) { // checks if all the hockServer APIs were called
+        apiHockInstance.done(function (error) { // checks if all the apiHockServer APIs were called
             expect(!error).to.be.ok();
-            done();
+
+            awsHockInstance.done(function (error) {
+                expect(!error).to.be.ok();
+                done();
+            });
         });
     });
 

src/routes/test/backups-test.js

@@ -119,8 +119,8 @@ describe('Backups API', function () {
 
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
-                        .put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] })
-                        .reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             request.post(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })

src/routes/test/clients-test.js

@@ -591,8 +591,8 @@ describe('Clients', function () {
         email: 'some@email.com',
         admin: true,
         salt: 'somesalt',
-        createdAt: (new Date()).toUTCString(),
-        modifiedAt: (new Date()).toUTCString(),
+        createdAt: (new Date()).toISOString(),
+        modifiedAt: (new Date()).toISOString(),
         resetToken: hat(256)
     };
 

src/routes/test/cloudron-test.js

@@ -450,8 +450,20 @@ describe('Cloudron', function () {
         });
 
         it('fails when in wrong state', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(409, {});
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) { 
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                    return body.size && body.region && body.restoreKey;
+                }).reply(409, {});
 
             injectShellMock();
 
@@ -463,7 +475,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }
@@ -477,8 +489,19 @@ describe('Cloudron', function () {
 
 
         it('succeeds', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(202, {});
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                return body.size && body.region && body.restoreKey;
+            }).reply(202, {});
+
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) { 
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             injectShellMock();
 
@@ -490,7 +513,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }

src/scripts/backupapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: backup.sh <appid> <url> <key>"
+    echo "Usage: backupapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -22,6 +22,7 @@ readonly DATA_DIR="${HOME}/data"
 app_id="$1"
 backup_url="$2"
 backup_key="$3"
+session_token="$4"
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
@@ -31,9 +32,17 @@ btrfs subvolume snapshot -r "${app_data_dir}" "${app_data_snapshot}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${app_data_snapshot}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT "${headers[@]}" --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/backupbox.sh

@@ -13,12 +13,13 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 2 ]; then
-    echo "Usage: backupbox.sh <url> <key>"
+    echo "Usage: backupbox.sh <url> <key> [aws session token]"
     exit 1
 fi
 
 backup_url="$1"
 backup_key="$2"
+session_token="$3"
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"
@@ -32,9 +33,17 @@ btrfs subvolume snapshot -r "${BOX_DATA_DIR}" "${box_snapshot_dir}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${box_snapshot_dir}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/collectlogs.sh

@@ -21,7 +21,7 @@ readonly program_name=$1
 
 echo "${program_name}.log"
 echo "-------------------"
-tail --lines=100 /var/log/supervisor/${program_name}.log
+journalctl --no-pager -u ${program_name} -n 100
 echo
 echo
 echo "dmesg"
@@ -31,7 +31,7 @@ echo
 echo
 echo "docker"
 echo "------"
-tail --lines=100 /var/log/upstart/docker.log
+journalctl --no-pager -u docker -n 50
 echo
 echo
 

src/scripts/reloadnginx.sh

@@ -12,12 +12,6 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${OSTYPE}" == "darwin"* ]]; then
-    # On Mac, brew installs supervisor in /usr/local/bin
-    export PATH=$PATH:/usr/local/bin
-fi
-
 if [[ "${BOX_ENV}" == "cloudron" ]]; then
     nginx -s reload
 fi
-

src/scripts/restoreapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: restoreapp.sh <appid> <url> <key>"
+    echo "Usage: restoreapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -23,6 +23,7 @@ readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max
 app_id="$1"
 restore_url="$2"
 restore_key="$3"
+session_token="$4"
 
 echo "Downloading backup: ${restore_url} and key: ${restore_key}"
 
@@ -30,7 +31,14 @@ for try in `seq 1 5`; do
     echo "Download backup from ${restore_url} (try ${try})"
     error_log=$(mktemp)
 
-    if $curl -L "${restore_url}" \
+    headers=("") # empty element required (http://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u)
+
+    # federated tokens in CaaS case need session token
+    if [[ ! -z "${session_token}" ]]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
+    if $curl -L "${headers[@]}" "${restore_url}" \
         | openssl aes-256-cbc -d -pass "pass:${restore_key}" \
         | tar -zxf - -C "${DATA_DIR}/${app_id}" 2>"${error_log}"; then
         chown -R yellowtent:yellowtent "${DATA_DIR}/${app_id}"

src/server.js

@@ -97,7 +97,7 @@ function initializeExpressSync() {
     // private routes
     router.get ('/api/v1/cloudron/config', rootScope, routes.cloudron.getConfig);
     router.post('/api/v1/cloudron/update', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
-    router.get ('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
+    router.post('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
     router.post('/api/v1/cloudron/migrate', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.migrate);
     router.post('/api/v1/cloudron/certificate', rootScope, multipart, routes.cloudron.setCertificate);
     router.get ('/api/v1/cloudron/graphs', rootScope, routes.graphs.getGraphs);
@@ -199,6 +199,7 @@ function initializeExpressSync() {
     return httpServer;
 }
 
+// provides hooks for the 'installer'
 function initializeInternalExpressSync() {
     var app = express();
     var httpServer = http.createServer(app);

src/settings.js

@@ -42,12 +42,9 @@ var assert = require('assert'),
     _ = require('underscore');
 
 var gDefaults = (function () {
-    var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
-    tz = tz ? tz.trim() : 'America/Los_Angeles';
-
     var result = { };
     result[exports.AUTOUPDATE_PATTERN_KEY] = '00 00 1,3,5,23 * * *';
-    result[exports.TIME_ZONE_KEY] = tz;
+    result[exports.TIME_ZONE_KEY] = 'America/Los_Angeles';
     result[exports.CLOUDRON_NAME_KEY] = 'Cloudron';
     result[exports.DEVELOPER_MODE_KEY] = false;
 

src/subdomainerror.js

@@ -0,0 +1,33 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    util = require('util');
+
+exports = module.exports = SubdomainError;
+
+function SubdomainError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(SubdomainError, Error);
+
+SubdomainError.NOT_FOUND = 'No such domain';
+SubdomainError.EXTERNAL_ERROR = 'External error';
+SubdomainError.STILL_BUSY = 'Still busy';
+SubdomainError.MISSING_CREDENTIALS = 'Missing credentials';

src/subdomains.js

@@ -0,0 +1,87 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    aws = require('./aws.js'),
+    caas = require('./caas.js'),
+    config = require('./config.js'),
+    debug = require('debug')('box:subdomains'),
+    util = require('util'),
+    SubdomainError = require('./subdomainerror.js');
+
+module.exports = exports = {
+    add: add,
+    addMany: addMany,
+    remove: remove,
+    status: status
+};
+
+// choose which subdomain backend we use
+// for test purpose we use aws
+function api() {
+    return config.token() && !config.TEST ? caas : aws;
+}
+
+function add(record, callback) {
+    assert.strictEqual(typeof record, 'object');
+    assert.strictEqual(typeof record.subdomain, 'string');
+    assert.strictEqual(typeof record.type, 'string');
+    assert.strictEqual(typeof record.value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: ', record);
+
+    api().addSubdomain(config.zoneName(), record.subdomain, record.type, record.value, function (error, changeId) {
+        if (error) return callback(error);
+        callback(null, changeId);
+    });
+}
+
+function addMany(records, callback) {
+    assert(util.isArray(records));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addMany: ', records);
+
+    var changeIds = [];
+
+    async.eachSeries(records, function (record, callback) {
+        add(record, function (error, changeId) {
+            if (error) return callback(error);
+
+            changeIds.push(changeId);
+
+            callback(null);
+        });
+    }, function (error) {
+        if (error) return callback(error);
+        callback(null, changeIds);
+    });
+}
+
+function remove(record, callback) {
+    assert.strictEqual(typeof record, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('remove: ', record);
+
+    api().delSubdomain(config.zoneName(), record.subdomain, record.type, record.value, function (error) {
+        if (error && error.reason !== SubdomainError.NOT_FOUND) return callback(error);
+
+        debug('deleteSubdomain: successfully deleted subdomain from aws.');
+
+        callback(null);
+    });
+}
+
+function status(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    api().getChangeStatus(changeId, function (error, status) {
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error));
+        callback(null, status === 'INSYNC' ? 'done' : 'pending');
+    });
+}

src/taskmanager.js

@@ -17,10 +17,7 @@ var appdb = require('./appdb.js'),
 var gActiveTasks = { };
 var gPendingTasks = [ ];
 
-// Task concurrency is 1 for two reasons:
-// 1. The backup scripts (app and box) turn off swap after finish disregarding other backup processes
-// 2. apptask getFreePort has race with multiprocess
-var TASK_CONCURRENCY = 1;
+var TASK_CONCURRENCY = 5;
 var NOOP_CALLBACK = function (error) { console.error(error); };
 
 function initialize(callback) {
@@ -31,6 +28,8 @@ function initialize(callback) {
         if (error) return callback(error);
 
         apps.forEach(function (app) {
+            if (app.installationState === appdb.ISTATE_INSTALLED && app.runState === appdb.RSTATE_RUNNING) return;
+
             debug('Creating process for %s (%s) with state %s', app.location, app.id, app.installationState);
             startAppTask(app.id);
         });
@@ -54,7 +53,8 @@ function uninitialize(callback) {
 
 function startNextTask() {
     if (gPendingTasks.length === 0) return;
-    assert.strictEqual(Object.keys(gActiveTasks).length, 0); // since we allow only one task at a time
+
+    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     startAppTask(gPendingTasks.shift());
 }
@@ -63,14 +63,20 @@ function startAppTask(appId) {
     assert.strictEqual(typeof appId, 'string');
     assert(!(appId in gActiveTasks));
 
-    var lockError = locker.lock(locker.OP_APPTASK);
-
-    if (lockError || Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
+    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug('Reached concurrency limit, queueing task for %s', appId);
         gPendingTasks.push(appId);
         return;
     }
 
+    var lockError = locker.recursiveLock(locker.OP_APPTASK);
+
+    if (lockError) {
+        debug('Locked for another operation, queueing task for %s', appId);
+        gPendingTasks.push(appId);
+        return;
+    }
+
     gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
     gActiveTasks[appId].once('exit', function (code) {
         debug('Task for %s completed with status %s', appId, code);

src/test/apptask-test.js

@@ -13,10 +13,10 @@ var addons = require('../addons.js'),
     database = require('../database.js'),
     expect = require('expect.js'),
     fs = require('fs'),
+    js2xml = require('js2xmlparser'),
     net = require('net'),
     nock = require('nock'),
     paths = require('../paths.js'),
-    sysinfo = require('../sysinfo.js'),
     _ = require('underscore');
 
 var MANIFEST = {
@@ -61,6 +61,21 @@ var APP = {
     dnsRecordId: 'someDnsRecordId'
 };
 
+ var awsHostedZones = {
+     HostedZones: [{
+         Id: '/hostedzone/ZONEID',
+         Name: 'localhost.',
+         CallerReference: '305AFD59-9D73-4502-B020-F4E6F889CB30',
+         ResourceRecordSetCount: 2,
+         ChangeInfo: {
+             Id: '/change/CKRTFJA0ANHXB',
+             Status: 'INSYNC'
+         }
+     }],
+    IsTruncated: false,
+    MaxItems: '100'
+ };
+
 describe('apptask', function () {
     before(function (done) {
         config.set('version', '0.5.0');
@@ -154,7 +169,7 @@ describe('apptask', function () {
     it('barfs on bad manifest', function (done) {
         var badApp = _.extend({ }, APP);
         badApp.manifest = _.extend({ }, APP.manifest);
-        delete badApp.manifest['id'];
+        delete badApp.manifest.id;
 
         apptask._verifyManifest(badApp, function (error) {
             expect(error).to.be.ok();
@@ -185,12 +200,20 @@ describe('apptask', function () {
     it('registers subdomain', function (done) {
         nock.cleanAll();
         var scope = nock(config.apiServerOrigin())
-            .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP.location, type: 'A', value: sysinfo.getIp() } ] })
-            .reply(201, { ids: [ APP.dnsRecordId ] });
+            .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+            .times(2)
+            .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+        var awsScope = nock(config.aws().endpoint)
+            .get('/2013-04-01/hostedzone')
+            .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }))
+            .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+            .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'RRID', Status: 'INSYNC' } }));
 
         apptask._registerSubdomain(APP, function (error) {
             expect(error).to.be(null);
             expect(scope.isDone()).to.be.ok();
+            expect(awsScope.isDone()).to.be.ok();
             done();
         });
     });
@@ -198,12 +221,20 @@ describe('apptask', function () {
     it('unregisters subdomain', function (done) {
         nock.cleanAll();
         var scope = nock(config.apiServerOrigin())
-            .delete('/api/v1/subdomains/' + APP.dnsRecordId + '?token=' + config.token())
-            .reply(204, {});
+            .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+            .times(2)
+            .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
-        apptask._unregisterSubdomain(APP, function (error) {
+        var awsScope = nock(config.aws().endpoint)
+            .get('/2013-04-01/hostedzone')
+            .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }))
+            .post('/2013-04-01/hostedzone/ZONEID/rrset/')
+            .reply(200, js2xml('ChangeResourceRecordSetsResponse', { ChangeInfo: { Id: 'RRID', Status: 'INSYNC' } }));
+
+        apptask._unregisterSubdomain(APP, APP.location, function (error) {
             expect(error).to.be(null);
             expect(scope.isDone()).to.be.ok();
+            expect(awsScope.isDone()).to.be.ok();
             done();
         });
     });

src/test/ldap-test.js

@@ -6,8 +6,6 @@
 
 'use strict';
 
-require('supererror', { splatchError: true});
-
 var database = require('../database.js'),
     expect = require('expect.js'),
     EventEmitter = require('events').EventEmitter,

src/updatechecker.js

@@ -10,21 +10,15 @@ exports = module.exports = {
 };
 
 var apps = require('./apps.js'),
-    assert = require('assert'),
     async = require('async'),
     config = require('./config.js'),
     debug = require('debug')('box:updatechecker'),
-    fs = require('fs'),
     mailer = require('./mailer.js'),
-    path = require('path'),
-    paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     superagent = require('superagent'),
     util = require('util');
 
-var NOOP_CALLBACK = function (error) { console.error(error); };
-
 var gAppUpdateInfo = { }, // id -> update info { creationDate, manifest }
     gBoxUpdateInfo = null,
     gMailedUser =  { };
@@ -138,6 +132,7 @@ function checkAppUpdates() {
 
                 mailer.appUpdateAvailable(app, gAppUpdateInfo[id]);
                 gMailedUser[id] = true;
+                iteratorDone();
             });
         });
     });
@@ -157,4 +152,3 @@ function checkBoxUpdates() {
         }
     });
 }
-

src/user.js

@@ -134,7 +134,7 @@ function createUser(username, password, email, admin, invitor, callback) {
         crypto.pbkdf2(password, salt, CRYPTO_ITERATIONS, CRYPTO_KEY_LENGTH, function (error, derivedKey) {
             if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
-            var now = (new Date()).toUTCString();
+            var now = (new Date()).toISOString();
             var user = {
                 id: username,
                 username: username,
@@ -203,17 +203,17 @@ function verifyWithEmail(email, password, callback) {
     });
 }
 
-function removeUser(username, callback) {
-    assert.strictEqual(typeof username, 'string');
+function removeUser(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.del(username, function (error) {
+    userdb.del(userId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
         callback(null);
 
-        mailer.userRemoved(username);
+        mailer.userRemoved(userId);
     });
 }
 
@@ -331,7 +331,7 @@ function setPassword(userId, newPassword, callback) {
         crypto.pbkdf2(newPassword, saltBuffer, CRYPTO_ITERATIONS, CRYPTO_KEY_LENGTH, function (error, derivedKey) {
             if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
-            user.modifiedAt = (new Date()).toUTCString();
+            user.modifiedAt = (new Date()).toISOString();
             user.password = new Buffer(derivedKey, 'binary').toString('hex');
             user.resetToken = '';
 

src/webhooks.js

@@ -0,0 +1,47 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    backupDone: backupDone
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:webhooks'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function backupDone(filename, app, appBackupIds, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert(!app || typeof app === 'object');
+    assert(!appBackupIds || util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('backupDone():', filename);
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupDone';
+        var data = {
+            boxVersion: config.version(),
+            restoreKey: filename,
+            appId: app ? app.id : null,
+            appVersion: app ? app.manifest.version : null,
+            appBackupIds: appBackupIds
+        };
+
+        superagent.post(url).send(data).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 200) return callback(new Error(result.text));
+            if (!result.body) return callback(new Error('Unexpected response'));
+
+            debug('backupDone()', filename);
+
+            return callback(null);
+        });
+    } else {
+        // TODO call custom webhook
+        callback(null);
+    }
+}

webadmin/src/appstatus.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron App Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/error.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/index.html

@@ -8,6 +8,11 @@
 
     <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
 
+    <!-- CSS -->
+    <link rel="stylesheet" type="text/css" href="/3rdparty/slick.css"/>
+    <link rel="stylesheet" type="text/css" href="/3rdparty/angular-ui-notification.min.css"/>
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -22,12 +27,8 @@
     <script src="3rdparty/js/bootstrap.min.js"></script>
 
     <!-- Slick carousel -->
-    <link rel="stylesheet" type="text/css" href="/3rdparty/slick.css"/>
     <script type="text/javascript" src="3rdparty/js/slick.js"></script>
 
-    <!-- Additional stylesheets -->
-    <link rel="stylesheet" type="text/css" href="/3rdparty/angular-ui-notification.min.css"/>
-
     <!-- Angularjs scripts -->
     <script src="3rdparty/js/angular.min.js"></script>
     <script src="3rdparty/js/angular-loader.min.js"></script>
@@ -48,9 +49,6 @@
     <!-- Main Application -->
     <script src="js/index.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body>
@@ -75,31 +73,38 @@
                         <br/>
                         <br/>
                     </div>
-                    <p>This update will install version <b>{{config.update.box.version}}</b> on your Cloudron.</p>
-                    <p>Recent Changes:</p>
-                    <ul>
-                        <li ng-repeat="change in config.update.box.changelog">{{change}}</li>
-                    </ul>
-                    <br/>
-                    <fieldset>
-                        <form name="update_form" role="form" ng-submit="doUpdate()" autocomplete="off">
-                            <div class="form-group" ng-class="{ 'has-error': update_form.password.$dirty && update_form.password.$invalid }">
-                                <label class="control-label" for="inputUpdatePassword">Give your password to verify that you are performing that action</label>
-                                <div class="control-label" ng-show="(!update_form.password.$dirty && update.error.password) || (update_form.password.$dirty && update_form.password.$invalid)">
-                                    <small ng-show="update_form.password.$error.required && !update.error.password">A password is required</small>
-                                    <small ng-show="update_form.password.$error.minlength">The password is too short</small>
-                                    <small ng-show="update_form.password.$error.maxlength">The password is too long</small>
-                                    <small ng-show="update.error.password">Incorrect password</small>
+                    <div ng-show="installedApps | readyToUpdate">
+                        <b ng-show="config.update.box.upgrade" class="text-danger">
+                            The update is a base system upgrade.<br/>
+                            This will cause some application downtime!<br/>
+                            <br/>
+                        </b>
+                        <p>New version: <b>{{config.update.box.version}}</b></p>
+                        <p>Recent Changes:</p>
+                        <ul>
+                            <li ng-repeat="change in config.update.box.changelog">{{change}}</li>
+                        </ul>
+                        <br/>
+                        <fieldset>
+                            <form name="update_form" role="form" ng-submit="doUpdate()" autocomplete="off">
+                                <div class="form-group" ng-class="{ 'has-error': update_form.password.$dirty && update_form.password.$invalid }">
+                                    <label class="control-label" for="inputUpdatePassword">Give your password to verify that you are performing that action</label>
+                                    <div class="control-label" ng-show="(!update_form.password.$dirty && update.error.password) || (update_form.password.$dirty && update_form.password.$invalid)">
+                                        <small ng-show="update_form.password.$error.required && !update.error.password">A password is required</small>
+                                        <small ng-show="update_form.password.$error.minlength">The password is too short</small>
+                                        <small ng-show="update_form.password.$error.maxlength">The password is too long</small>
+                                        <small ng-show="update.error.password">Incorrect password</small>
+                                    </div>
+                                    <input type="password" class="form-control" ng-model="update.password" id="inputUpdatePassword" name="password" placeholder="Password" ng-maxlength="512" ng-minlength="5" required autofocus>
                                 </div>
-                                <input type="password" class="form-control" ng-model="update.password" id="inputUpdatePassword" name="password" placeholder="Password" ng-maxlength="512" ng-minlength="5" required autofocus>
-                            </div>
-                            <input class="ng-hide" type="submit" ng-disabled="update_form.$invalid || update.busy"/>
-                        </form>
-                    </fieldset>
+                                <input class="ng-hide" type="submit" ng-disabled="update_form.$invalid || update.busy"/>
+                            </form>
+                        </fieldset>
+                    </div>
                 </div>
                 <div class="modal-footer">
                     <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
+                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy" ng-show="installedApps | readyToUpdate"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
                 </div>
             </div>
         </div>
@@ -144,7 +149,6 @@
                                 <li><a href="#/account"><i class="fa fa-user fa-fw"></i> Account</a></li>
                                 <li ng-show="user.admin && config.isDev"><a href="#/dns"><i class="fa fa-wrench fa-fw"></i> DNS Management</a></li>
                                 <li ng-show="user.admin"><a href="#/graphs"><i class="fa fa-bar-chart fa-fw"></i> Graphs</a></li>
-                                <li ng-show="user.admin"><a href="#/upgrade"><i class="fa fa-arrow-up fa-fw"></i> Upgrade</a></li>
                                 <li><a href="#/support"><i class="fa fa-comment fa-fw"></i> Support</a></li>
                                 <li ng-show="user.admin"><a href="#/settings"><i class="fa fa-wrench fa-fw"></i> Settings</a></li>
                                 <li class="divider"></li>

webadmin/src/js/client.js

@@ -23,7 +23,7 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
 
     function defaultErrorHandler(callback) {
         return function (data, status) {
-            if (status === 401) return client.logout();
+            if (status === 401) return client.login();
             if (status === 503) {
                 // this could indicate a update/upgrade/restore/migration
                 client.progress(function (error, result) {
@@ -425,7 +425,7 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
     };
 
     Client.prototype.reboot = function (callback) {
-        $http.get(client.apiOrigin + '/api/v1/cloudron/reboot').success(function(data, status) {
+        $http.post(client.apiOrigin + '/api/v1/cloudron/reboot', { }).success(function(data, status) {
             if (status !== 202 || typeof data !== 'object') return callback(new ClientError(status, data));
             callback(null, data);
         }).error(defaultErrorHandler(callback));
@@ -605,12 +605,27 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
         });
     };
 
+    Client.prototype.login = function () {
+        this.setToken(null);
+        this._userInfo = {};
+
+        var callbackURL = window.location.protocol + '//' + window.location.host + '/login_callback.html';
+        var scope = 'root,profile,apps,roleAdmin';
+
+        // generate a state id to protect agains csrf
+        var state = Math.floor((1 + Math.random()) * 0x1000000000000).toString(16).substring(1);
+        window.localStorage.oauth2State = state;
+
+        window.location.href = this.apiOrigin + '/api/v1/oauth/dialog/authorize?response_type=token&client_id=' + this._clientId + '&redirect_uri=' + callbackURL + '&scope=' + scope + '&state=' + state;
+    };
+
     Client.prototype.logout = function () {
         this.setToken(null);
         this._userInfo = {};
 
         // logout from OAuth session
-        window.location.href = client.apiOrigin + '/api/v1/session/logout';
+        var origin = window.location.protocol + "//" + window.location.host;
+        window.location.href = this.apiOrigin + '/api/v1/session/logout?redirect=' + origin;
     };
 
     Client.prototype.exchangeCodeForToken = function (code, callback) {

webadmin/src/js/index.js

@@ -40,9 +40,6 @@ app.config(['$routeProvider', function ($routeProvider) {
     }).when('/support', {
         controller: 'SupportController',
         templateUrl: 'views/support.html'
-    }).when('/upgrade', {
-        controller: 'UpgradeController',
-        templateUrl: 'views/upgrade.html'
     }).otherwise({ redirectTo: '/'});
 }]);
 
@@ -96,13 +93,13 @@ app.filter('installationStateLabel', function() {
         var waiting = app.progress === 0 ? ' (Waiting)' : '';
 
         switch (app.installationState) {
-        case ISTATES.PENDING_INSTALL: return 'Installing...' + waiting;
-        case ISTATES.PENDING_CONFIGURE: return 'Configuring...' + waiting;
-        case ISTATES.PENDING_UNINSTALL: return 'Uninstalling...' + waiting;
-        case ISTATES.PENDING_RESTORE: return 'Restoring...' + waiting;
-        case ISTATES.PENDING_UPDATE: return 'Updating...' + waiting;
-        case ISTATES.PENDING_FORCE_UPDATE: return 'Updating...' + waiting;
-        case ISTATES.PENDING_BACKUP: return 'Backing up...' + waiting;
+        case ISTATES.PENDING_INSTALL: return 'Installing' + waiting;
+        case ISTATES.PENDING_CONFIGURE: return 'Configuring' + waiting;
+        case ISTATES.PENDING_UNINSTALL: return 'Uninstalling' + waiting;
+        case ISTATES.PENDING_RESTORE: return 'Restoring' + waiting;
+        case ISTATES.PENDING_UPDATE: return 'Updating' + waiting;
+        case ISTATES.PENDING_FORCE_UPDATE: return 'Updating' + waiting;
+        case ISTATES.PENDING_BACKUP: return 'Backing up' + waiting;
         case ISTATES.ERROR: return 'Error';
         case ISTATES.INSTALLED: {
             if (app.runState === 'running') {

webadmin/src/js/main.js

@@ -23,17 +23,6 @@ angular.module('Application').controller('MainController', ['$scope', '$route',
         Client.logout();
     };
 
-    $scope.login = function () {
-        var callbackURL = window.location.protocol + '//' + window.location.host + '/login_callback.html';
-        var scope = 'root,profile,apps,roleAdmin';
-
-        // generate a state id to protect agains csrf
-        var state = Math.floor((1 + Math.random()) * 0x1000000000000).toString(16).substring(1);
-        window.localStorage.oauth2State = state;
-
-        window.location.href = Client.apiOrigin + '/api/v1/oauth/dialog/authorize?response_type=token&client_id=' + Client._clientId + '&redirect_uri=' + callbackURL + '&scope=' + scope + '&state=' + state;
-    };
-
     $scope.setup = function () {
         window.location.href = '/error.html?errorCode=1';
     };
@@ -78,43 +67,43 @@ angular.module('Application').controller('MainController', ['$scope', '$route',
         if (error) return $scope.error(error);
         if (isFirstTime) return $scope.setup();
 
-        // we use the config request as an indicator if the token is still valid
-        // TODO we should probably attach such a handler for each request, as the token can get invalid
-        // at any time!
-        if (localStorage.token) {
-            Client.refreshConfig(function (error) {
-                if (error && error.statusCode === 401) return $scope.login();
+        Client.refreshConfig(function (error) {
+            if (error) return $scope.error(error);
+
+            // check version and force reload if needed
+            if (!localStorage.version) {
+                localStorage.version = Client.getConfig().version;
+            } else if (localStorage.version !== Client.getConfig().version) {
+                localStorage.version = Client.getConfig().version;
+                window.location.reload(true);
+            }
+
+            Client.refreshUserInfo(function (error, result) {
                 if (error) return $scope.error(error);
 
-                Client.refreshUserInfo(function (error, result) {
+                Client.refreshInstalledApps(function (error) {
                     if (error) return $scope.error(error);
 
-                    Client.refreshInstalledApps(function (error) {
-                        if (error) return $scope.error(error);
+                    // kick off installed apps and config polling
+                    var refreshAppsTimer = $interval(Client.refreshInstalledApps.bind(Client), 2000);
+                    var refreshConfigTimer = $interval(Client.refreshConfig.bind(Client), 5000);
+                    var refreshUserInfoTimer = $interval(Client.refreshUserInfo.bind(Client), 5000);
 
-                        // kick off installed apps and config polling
-                        var refreshAppsTimer = $interval(Client.refreshInstalledApps.bind(Client), 2000);
-                        var refreshConfigTimer = $interval(Client.refreshConfig.bind(Client), 5000);
-                        var refreshUserInfoTimer = $interval(Client.refreshUserInfo.bind(Client), 5000);
-
-                        $scope.$on('$destroy', function () {
-                            $interval.cancel(refreshAppsTimer);
-                            $interval.cancel(refreshConfigTimer);
-                            $interval.cancel(refreshUserInfoTimer);
-                        });
-
-                        // now mark the Client to be ready
-                        Client.setReady();
-
-                        $scope.config = Client.getConfig();
-
-                        $scope.initialized = true;
+                    $scope.$on('$destroy', function () {
+                        $interval.cancel(refreshAppsTimer);
+                        $interval.cancel(refreshConfigTimer);
+                        $interval.cancel(refreshUserInfoTimer);
                     });
+
+                    // now mark the Client to be ready
+                    Client.setReady();
+
+                    $scope.config = Client.getConfig();
+
+                    $scope.initialized = true;
                 });
             });
-        } else {
-            $scope.login();
-        }
+        });
     });
 
     // wait till the view has loaded until showing a modal dialog

webadmin/src/nakeddomain.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/setup.html

@@ -6,6 +6,9 @@
 
     <title> Cloudron Setup </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -31,9 +34,6 @@
     <!-- Setup Application -->
     <script src="js/setup.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body class="setup" ng-app="Application" ng-controller="SetupController">

webadmin/src/theme.scss

@@ -120,7 +120,6 @@ html {
 .grid-item {
     padding: 10px;
     min-width: 200px;
-    overflow: hidden;
 }
 
 .grid-item:hover .grid-item-bottom {
@@ -175,6 +174,12 @@ html {
     }
 }
 
+.app-update-badge {
+    position: absolute;
+    right: 0;
+    top: 0;
+}
+
 // ----------------------------
 // Appstore view
 // ----------------------------
@@ -195,8 +200,8 @@ html {
 
 .appstore-item-badge-testing {
     position: absolute;
-    right: 15px;
-    top: 15px;
+    right: 0;
+    top: 2px;
 }
 
 .appstore-item-content-testing {
@@ -330,6 +335,10 @@ html {
     background-color: #5CB85C;
 }
 
+.badge-warning {
+    background-color: #EFBD48;
+}
+
 .badge-danger {
     background-color: $brand-danger;
 }
@@ -350,23 +359,6 @@ html {
     max-width: 800px;
 }
 
-.app-update-badge {
-    font-size: $font-size-h4;
-    position: absolute;
-    left: 2px;
-    top: 2px;
-    width: $font-size-h4 + 6px;
-    text-overflow: ellipsis;
-    white-space: nowrap;
-    overflow: hidden;
-    background-color: transparent;
-}
-
-.app-update-badge:hover {
-    width: inherit;
-    background-color: #5CB85C;
-}
-
 .text-success {
     color: #5CB85C;
 }

webadmin/src/update.html

@@ -6,6 +6,9 @@
 
     <title> Cloudron Update </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -23,9 +26,6 @@
     <!-- Update Application -->
     <script src="js/update.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body ng-app="Application" ng-controller="Controller" style="background-color: #7F7F7F">

webadmin/src/views/apps.html

@@ -244,28 +244,32 @@
                     </div>
                     <div class="grid-item-bottom" ng-show="user.admin">
                         <br/>
+                        <br/>
 
                         <div>
-                            <a href="" ng-click="showUninstall(app)"><i class="fa fa-remove scale"></i></a>
+                            <a href="" ng-click="showUninstall(app)" title="Uninstall App"><i class="fa fa-remove scale"></i></a>
                         </div>
 
                         <div ng-show="(app | installError) === true">
-                            <a href="" ng-click="showRestore(app)"><i class="fa fa-undo scale"></i></a>
+                            <a href="" ng-click="showRestore(app)" title="Restore App"><i class="fa fa-undo scale"></i></a>
                         </div>
 
                         <div ng-show="(app | installSuccess) == true">
-                            <a href="" ng-click="showConfigure(app)"><i class="fa fa-wrench scale"></i></a>
-                        </div>
-
-                        <!-- we check the version here because the box updater does not know when an app gets updated -->
-                        <div ng-show="config.update.apps[app.id].manifest.version && config.update.apps[app.id].manifest.version !== app.manifest.version && (app | installSuccess)">
-                            <a href="" ng-click="showUpdate(app)"><i class="fa fa-arrow-up text-success scale"></i></a>
+                            <a href="" ng-click="showConfigure(app)" title="Configure App"><i class="fa fa-wrench scale"></i></a>
                         </div>
 
                         <br/>
                     </div>
+
+                    <!-- we check the version here because the box updater does not know when an app gets updated -->
+                    <div class="app-update-badge" ng-show="config.update.apps[app.id].manifest.version && config.update.apps[app.id].manifest.version !== app.manifest.version && (app | installSuccess)">
+                        <a href="" ng-click="showUpdate(app)" title="Update Available"><i class="fa fa-asterisk fa-2x text-success scale"></i></a>
+                    </div>
                 </a>
             </div>
         </div>
     </div>
 </div>
+
+<!-- Offset the footer -->
+<br/><br/>

webadmin/src/views/appstore.html

@@ -147,8 +147,9 @@
         <div class="col-md-10" ng-show="ready && apps.length">
             <div class="row-no-margin">
                 <div class="col-sm-1 appstore-item" ng-repeat="app in apps">
-                    <div class="appstore-item-content highlight" ng-click="showInstall(app)" ng-class="{ 'appstore-item-content-testing': app.publishState === 'testing' }">
+                    <div class="appstore-item-content highlight" ng-click="showInstall(app)" ng-class="{ 'appstore-item-content-testing': (app.publishState === 'testing' || app.publishState === 'pending_approval') }">
                         <span class="badge badge-danger appstore-item-badge-testing" ng-show="app.publishState === 'testing'">Testing</span>
+                        <span class="badge badge-warning appstore-item-badge-testing" ng-show="app.publishState === 'pending_approval'">Pending Approval</span>
                         <div class="appstore-item-content-icon col-same-height">
                             <img ng-src="{{app.iconUrl}}" onerror="this.onerror=null;this.src='img/appicon_fallback.png'" class="app-icon"/>
                         </div>

webadmin/src/views/graphs.html

@@ -14,39 +14,15 @@
     <div class="row shadow memory-app-container">
         <h2>Disk Usage</h2>
         <br/>
-        <div class="col-md-4">
-            <h4>Applications <span class="badge">{{ diskUsage['docker'].sum }} GB</span></h4>
-            <canvas id="dockerDiskUsageChart" width="200" height="200"></canvas>
-            <p>
-                <span class="text-success">Free {{ diskUsage['docker'].free }} GB</span>
-                &nbsp;
-                <span class="text-warning">Reserved {{ diskUsage['docker'].reserved }} GB</span>
-                &nbsp;
-                <span class="text-primary">Used {{ diskUsage['docker'].used }} GB</span>
-            </p>
-        </div>
-        <div class="col-md-4">
+        <div class="col-md-offset-4 col-md-4">
             <h4>Data <span class="badge">{{ diskUsage['box'].sum }} GB</span></h4>
             <canvas id="boxDiskUsageChart" width="200" height="200"></canvas>
             <p>
                 <span class="text-success">Free {{ diskUsage['box'].free }} GB</span>
                 &nbsp;
-                <span class="text-warning">Reserved {{ diskUsage['box'].reserved }} GB</span>
-                &nbsp;
                 <span class="text-primary">Used {{ diskUsage['box'].used }} GB</span>
             </p>
         </div>
-        <div class="col-md-4">
-            <h4>System (all) <span class="badge">{{ diskUsage['cloudron'].sum }} GB</span></h4>
-            <canvas id="cloudronDiskUsageChart" width="200" height="200"></canvas>
-            <p>
-                <span class="text-success">Free {{ diskUsage['cloudron'].free }} GB</span>
-                &nbsp;
-                <span class="text-warning">Reserved {{ diskUsage['cloudron'].reserved }} GB</span>
-                &nbsp;
-                <span class="text-primary">Used {{ diskUsage['cloudron'].used }} GB</span>
-            </p>
-        </div>
     </div>
 
     <br/>

webadmin/src/views/graphs.js

@@ -33,8 +33,7 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
 
     function renderDisk(type, free, reserved, used) {
         $scope.diskUsage[type] = {
-            used: bytesToGigaBytes(used.datapoints[0][0]),
-            reserved: bytesToGigaBytes(reserved.datapoints[0][0]),
+            used: bytesToGigaBytes(used.datapoints[0][0] + reserved.datapoints[0][0]),
             free: bytesToGigaBytes(free.datapoints[0][0]),
             sum: bytesToGigaBytes(used.datapoints[0][0] + reserved.datapoints[0][0] + free.datapoints[0][0])
         };
@@ -44,11 +43,6 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
             color: "#2196F3",
             highlight: "#82C4F8",
             label: "Used"
-        }, {
-            value: $scope.diskUsage[type].reserved,
-            color: "#f0ad4e",
-            highlight: "#F8D9AC",
-            label: "Reserved"
         }, {
             value: $scope.diskUsage[type].free,
             color:"#27CE65",
@@ -98,7 +92,7 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
             var options = {
                 scaleOverride: true,
                 scaleSteps: 10,
-                scaleStepWidth: $scope.activeApp === 'system' ? 100 : 10,
+                scaleStepWidth: $scope.activeApp === 'system' ? 200 : ($scope.activeApp.manifest.memoryLimit ? parseInt($scope.activeApp.manifest.memoryLimit/10000000,10) : 20),
                 scaleStartValue: 0
             };
 
@@ -111,21 +105,11 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
         Client.graphs([
             'averageSeries(collectd.localhost.df-loop0.df_complex-free)',
             'averageSeries(collectd.localhost.df-loop0.df_complex-reserved)',
-            'averageSeries(collectd.localhost.df-loop0.df_complex-used)',
-
-            'averageSeries(collectd.localhost.df-loop1.df_complex-free)',
-            'averageSeries(collectd.localhost.df-loop1.df_complex-reserved)',
-            'averageSeries(collectd.localhost.df-loop1.df_complex-used)',
-
-            'averageSeries(collectd.localhost.df-vda1.df_complex-free)',
-            'averageSeries(collectd.localhost.df-vda1.df_complex-reserved)',
-            'averageSeries(collectd.localhost.df-vda1.df_complex-used)',
+            'averageSeries(collectd.localhost.df-loop0.df_complex-used)'
         ], '-1min', function (error, data) {
             if (error) return console.log(error);
 
-            renderDisk('docker', data[0], data[1], data[2]);
-            renderDisk('box', data[3], data[4], data[5]);
-            renderDisk('cloudron', data[6], data[7], data[8]);
+            renderDisk('box', data[0], data[1], data[2]);
         });
     };
 

webadmin/src/views/upgrade.html

@@ -1,135 +0,0 @@
-<!-- Modal upgrade -->
-<div class="modal fade" id="upgradeModal" tabindex="-1" role="dialog">
-    <div class="modal-dialog">
-        <div class="modal-content">
-            <div class="modal-header">
-                <h4 class="modal-title">Upgrade your Cloudron</h4>
-            </div>
-            <div class="modal-body">
-                Do you really want to upgrade your Cloudron?<br/>
-                <br/>
-                <span class="text-danger">Your Cloudron will have a downtime of around 10 minutes, where none of you apps will be reachable.</span>
-                <br/>
-                <br/>
-                <form name="upgradeForm" class="form-signin" role="form" novalidate ng-submit="upgrade()" autocomplete="off">
-                    <fieldset>
-                        <div class="form-group" ng-class="{ 'has-error': (upgradeForm.password.$dirty && upgradeForm.password.$invalid) || (!upgradeForm.password.$dirty && upgrade.error.password) }">
-                            <label class="control-label" for="upgradePasswordInput">Provide your password to confirm this action</label>
-                            <input type="password" class="form-control" ng-model="upgrade.password" id="upgradePasswordInput" name="password" ng-maxlength="512" ng-minlength="5" required autofocus autocomplete="off">
-                        </div>
-                        <input class="ng-hide" type="submit" ng-disabled="upgradeForm.$invalid || busy"/>
-                    </fieldset>
-                </form>
-            </div>
-            <div class="modal-footer">
-                <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
-                <button type="button" class="btn btn-success" ng-click="upgrade()" ng-disabled="upgradeForm.$invalid || busy"><i class="fa fa-spinner fa-pulse" ng-show="busy"></i> Upgrade</button>
-            </div>
-        </div>
-    </div>
-</div>
-
-<!-- Modal relocate -->
-<div class="modal fade" id="relocationModal" tabindex="-1" role="dialog">
-    <div class="modal-dialog">
-        <div class="modal-content">
-            <div class="modal-header">
-                <h4 class="modal-title">Relocate your Cloudron</h4>
-            </div>
-            <div class="modal-body">
-                Do you really want to relocate your Cloudron from <b>{{currentRegionSlug}}</b> to <b>{{relocation.region.slug}}</b><br/>
-                <br/>
-                <span class="text-danger">Your Cloudron will have a downtime of around 10 minutes, where none of you apps will be reachable.</span>
-                <br/>
-                <br/>
-                <form name="relocateForm" class="form-signin" role="form" novalidate ng-submit="relocate()" autocomplete="off">
-                    <fieldset>
-                        <div class="form-group" ng-class="{ 'has-error': (relocateForm.password.$dirty && relocateForm.password.$invalid) || (!relocateForm.password.$dirty && relocation.error.password) }">
-                            <label class="control-label" for="relocationPasswordInput">Provide your password to confirm this action</label>
-                            <input type="password" class="form-control" ng-model="relocation.password" id="relocationPasswordInput" name="password" ng-maxlength="512" ng-minlength="5" required autofocus autocomplete="off">
-                        </div>
-                        <input class="ng-hide" type="submit" ng-disabled="relocateForm.$invalid || busy"/>
-                    </fieldset>
-                </form>
-            </div>
-            <div class="modal-footer">
-                <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
-                <button type="button" class="btn btn-success" ng-click="relocate()" ng-disabled="relocateForm.$invalid || busy"><i class="fa fa-spinner fa-pulse" ng-show="busy"></i> Relocate</button>
-            </div>
-        </div>
-    </div>
-</div>
-
-<br/>
-<br/>
-
-<div style="margin-bottom: 15px; text-align: center" ng-show="user.admin">
-    <div class="row" ng-show="availableSizes.length > 1">
-        <div class="col-md-12">
-            <h3>Choose a Model to upgrade to</h3>
-        </div>
-    </div>
-    <br/>
-    <div class="row" ng-show="availableSizes.length > 1">
-        <div class="col-md-6 col-md-offset-3">
-            <div class="cloudron-model-list">
-                <div class="cloudron-model-item" ng-repeat="size in availableSizes">
-                    <div class="cloudron-model-item-content shadow" ng-class="{ 'selected': size.slug === currentSize.slug }" style="height: {{ 120 + $index * 30 }}px">
-                        <!-- <img src="img/box.png" style="transform: scale({{ size.price/50.0 }});"/><br/> -->
-                        <h3>{{ size.name }}</h3>
-                        <h5>${{ (size.price/100).toFixed() }}/mo</h5>
-                        <button class="btn btn-success" ng-disabled="busy" ng-hide="size.slug === currentSize.slug" ng-click="showUpgradeConfirm(size)">Upgrade</button>
-                        <button class="btn btn-success" ng-show="size.slug === currentSize.slug" data-toggle="tooltip" data-placement="top" title="Your Current Model"><i class="fa fa-check"></i></button>
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-    <div class="row" ng-show="availableSizes.length > 1">
-        <div class="col-md-12">
-            <p>
-                Larger Cloudrons allow to run more apps and bring better performance to your installed services.
-            </p>
-        </div>
-    </div>
-    <div class="row" ng-show="availableSizes.length <= 1">
-        <div class="col-md-12">
-            <h4>You are already using the largest available Cloudron model.</h4>
-        </div>
-    </div>
-    <br/>
-    <br/>
-    <br/>
-    <br/>
-    <br/>
-    <div class="row">
-        <div class="form-group col-md-12">
-            <h3>Choose your Region, in case you want to relocate your Cloudron</h3>
-            <p>
-                The closer you are to your Cloudron, the faster the access speed will be.
-            </p>
-        </div>
-    </div>
-    <br/>
-    <div class="row">
-        <div class="form-group col-md-12">
-            <div class="region-select-map">
-                <div class="region-select" ng-click="setRegion('sfo1')" ng-class="{ 'region-selected': relocation.region.slug === 'sfo1' }" style="width: 270px; background-image: url('img/world_left.png');">
-                    <div class="region-pin region-pin-left" ng-class="{ 'region-pin-selected-left': relocation.region.slug === 'sfo1' }">
-                        <img src="img/pin.png" height="32px" width="16px"/> San Francisco
-                    </div>
-                </div>
-                <div class="region-select" ng-click="setRegion('ams3')" ng-class="{ 'region-selected': relocation.region.slug === 'ams3' }" style="width: 330px; background-image: url('img/world_right.png');">
-                    <div class="region-pin region-pin-right" ng-class="{ 'region-pin-selected-right': relocation.region.slug === 'ams3' }">
-                        <img src="img/pin.png" height="32px" width="16px"/> Amsterdam
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-    <div class="row">
-        <div class="form-group col-md-12">
-            <button class="btn btn-success" ng-click="showRelocationConfirm()" ng-disabled="(relocation.region.slug === currentRegionSlug) || busy">Relocate</button>
-        </div>
-    </div>
-</div>

webadmin/src/views/upgrade.js

@@ -1,114 +0,0 @@
-'use strict';
-
-angular.module('Application').controller('UpgradeController', ['$scope', '$location', 'Client', 'AppStore', function ($scope, $location, Client, AppStore) {
-    Client.onReady(function () { if (!Client.getUserInfo().admin) $location.path('/'); });
-
-    $scope.user = Client.getUserInfo();
-    $scope.config = Client.getConfig();
-    $scope.busy = false;
-    $scope.availableRegions = [];
-    $scope.availableSizes = [];
-
-    $scope.currentSize = null;
-    $scope.currentRegionSlug = null;
-
-    $scope.upgrade = {
-        size: null,
-        error: {},
-        password: null
-    };
-
-    $scope.relocation = {
-        region: null,
-        error: {},
-        password: null
-    };
-
-    $scope.showUpgradeConfirm = function (size) {
-        $scope.upgrade.size = size;
-        $('#upgradeModal').modal('show');
-    };
-
-    $scope.upgrade = function () {
-        $scope.busy = true;
-
-        Client.migrate($scope.upgrade.size.slug, $scope.currentRegionSlug, $scope.upgrade.password, function (error) {
-            $scope.busy = false;
-
-            if (error && error.statusCode === 403) {
-                $scope.upgrade.error.password = true;
-                $scope.upgrade.password = '';
-                $('#upgradePasswordInput').focus();
-                return;
-            } else if (error) {
-                return console.error(error);
-            }
-
-            $('#upgradeModal').modal('hide');
-        });
-    };
-
-    $scope.showRelocationConfirm = function () {
-        $('#relocationModal').modal('show');
-    };
-
-    $scope.relocate = function () {
-        $scope.busy = true;
-
-        Client.migrate($scope.currentSize.slug, $scope.relocation.region.slug, $scope.relocation.password, function (error) {
-            $scope.busy = false;
-
-            if (error && error.statusCode === 403) {
-                $scope.relocation.error.password = true;
-                $scope.relocation.password = '';
-                $('#relocationPasswordInput').focus();
-                return;
-            } else if (error) {
-                return console.error(error);
-            }
-
-            $('#relocationModal').modal('hide');
-        });
-    };
-
-    $scope.setRegion = function (regionSlug) {
-        $scope.availableRegions.forEach(function (region) {
-            if (region.slug.indexOf(regionSlug) === 0) $scope.relocation.region = region;
-        });
-    };
-
-    Client.onReady(function () {
-        AppStore.getSizes(function (error, result) {
-            if (error) return console.error(error);
-
-            // result array is ordered by size
-            var found = false;
-            result = result.filter(function (size) {
-                if (size.slug === $scope.config.size) {
-                    $scope.currentSize = size;
-                    found = true;
-                    return true;
-                } else {
-                    return found;
-                }
-            });
-            angular.copy(result, $scope.availableSizes);
-
-            AppStore.getRegions(function (error, result) {
-                if (error) return console.error(error);
-
-                angular.copy(result, $scope.availableRegions);
-
-                $scope.currentRegionSlug = $scope.config.region;
-                $scope.setRegion($scope.config.region);
-            });
-        });
-    });
-
-    // setup all the dialog focus handling
-    ['upgradeModal', 'relocationModal'].forEach(function (id) {
-        $('#' + id).on('shown.bs.modal', function () {
-            $(this).find("[autofocus]:first").focus();
-        });
-    });
-}]);