Actually hand the backupKey over in an update

This needs aws mock

.gitignore

@@ -4,10 +4,6 @@ docs/
 webadmin/dist/
 setup/splash/website/
 
-# vim swam files
+# vim swap files
 *.swp
 
-# supervisor
-supervisord.pid
-supervisord.log
-

README.md

@@ -4,7 +4,6 @@ The Box
 Development setup
 -----------------
 * sudo useradd -m yellowtent
-** This dummy user is required for supervisor 'box' configs
 ** Add admin-localhost as 127.0.0.1 in /etc/hosts
 ** All apps will be installed as hypened-subdomains of localhost. You should add
    hyphened-subdomains of your apps into /etc/hosts

crashnotifier.js

@@ -2,20 +2,12 @@
 
 'use strict';
 
-// WARNING This is a supervisor eventlistener!
-//         The communication happens via stdin/stdout
-//         !! No console.log() allowed
-//         !! Do not set DEBUG
-
 var assert = require('assert'),
     mailer = require('./src/mailer.js'),
     safe = require('safetydance'),
-    supervisor = require('supervisord-eventlistener'),
     path = require('path'),
     util = require('util');
 
-var gLastNotifyTime = {};
-var gCooldownTime = 1000 * 60  * 5; // 5 min
 var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
 
 function collectLogs(program, callback) {
@@ -26,28 +18,30 @@ function collectLogs(program, callback) {
     callback(null, logs);
 }
 
-supervisor.on('PROCESS_STATE_EXITED', function (headers, data) {
+function sendCrashNotification(processName) {
-    if (data.expected === '1') return console.error('Normal app %s exit', data.processname);
+    collectLogs(processName, function (error, result) {
-
-    console.error('%s exited unexpectedly', data.processname);
-
-    collectLogs(data.processname, function (error, result) {
         if (error) {
             console.error('Failed to collect logs.', error);
             result = util.format('Failed to collect logs.', error);
         }
 
-        if (!gLastNotifyTime[data.processname] || gLastNotifyTime[data.processname] < Date.now() - gCooldownTime) {
+        console.log('Sending crash notification email for', processName);
-            console.error('Send mail.');
+        mailer.sendCrashNotification(processName, result);
-            mailer.sendCrashNotification(data.processname, result);
-            gLastNotifyTime[data.processname] = Date.now();
-        } else {
-            console.error('Do not send mail, already sent one recently.');
-        }
     });
-});
+}
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    mailer.initialize(function (error) {
+        if (error) return console.error(error);
+
+        sendCrashNotification(processName);
+    });
+}
+
+main();
 
-mailer.initialize(function () {
-    supervisor.listen(process.stdin, process.stdout);
-    console.error('Crashnotifier listening...');
-});

npm-shrinkwrap.json

@@ -7,6 +7,28 @@
       "from": "https://registry.npmjs.org/async/-/async-1.2.1.tgz",
       "resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz"
     },
+    "aws-sdk": {
+      "version": "2.1.46",
+      "from": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "dependencies": {
+        "sax": {
+          "version": "0.5.3",
+          "from": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz",
+          "resolved": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz"
+        },
+        "xml2js": {
+          "version": "0.2.8",
+          "from": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz",
+          "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz"
+        },
+        "xmlbuilder": {
+          "version": "0.4.2",
+          "from": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz",
+          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz"
+        }
+      }
+    },
     "body-parser": {
       "version": "1.13.1",
       "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.13.1.tgz",
@@ -105,23 +127,24 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "1.6.0",
+      "version": "1.7.0",
-      "from": "cloudron-manifestformat@1.6.0",
+      "from": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
       "dependencies": {
         "java-packagename-regex": {
           "version": "1.0.0",
-          "from": "java-packagename-regex@>=1.0.0 <2.0.0",
+          "from": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz",
           "resolved": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz"
         },
         "safetydance": {
           "version": "0.0.15",
-          "from": "safetydance@0.0.15",
+          "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz",
           "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz"
         },
         "tv4": {
-          "version": "1.1.12",
+          "version": "1.2.3",
-          "from": "tv4@>=1.1.9 <2.0.0",
+          "from": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz",
-          "resolved": "http://registry.npmjs.org/tv4/-/tv4-1.1.12.tgz"
+          "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz"
         }
       }
     },
@@ -2418,11 +2441,6 @@
         }
       }
     },
-    "supervisord-eventlistener": {
-      "version": "0.1.0",
-      "from": "https://registry.npmjs.org/supervisord-eventlistener/-/supervisord-eventlistener-0.1.0.tgz",
-      "resolved": "https://registry.npmjs.org/supervisord-eventlistener/-/supervisord-eventlistener-0.1.0.tgz"
-    },
     "tail-stream": {
       "version": "0.2.1",
       "from": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",

package.json

@@ -17,8 +17,9 @@
   },
   "dependencies": {
     "async": "^1.2.1",
+    "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^1.6.0",
+    "cloudron-manifestformat": "^1.7.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
@@ -60,7 +61,6 @@
     "split": "^1.0.0",
     "superagent": "~0.21.0",
     "supererror": "^0.7.0",
-    "supervisord-eventlistener": "^0.1.0",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "underscore": "^1.7.0",
     "valid-url": "^1.0.9",
@@ -68,7 +68,6 @@
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "^2.1.10",
     "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",

setup/DESIGN.md

@@ -16,7 +16,7 @@ and replace it with a new one for an update.
 
 Because we do not package things as Docker yet, we should be careful
 about the code here. We have to expect remains of an older setup code.
-For example, older supervisor or nginx configs might be around.
+For example, older systemd or nginx configs might be around.
 
 The config directory is _part_ of the container and is not a VOLUME.
 Which is to say that the files will be nuked from one update to the next.
@@ -40,7 +40,7 @@ version (see below) or the mysql/postgresql data etc.
 
   * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
 
-  * supervisor is then started
+  * box services are then started
 
 setup_infra.sh
 This setups containers like graphite, mail and the addons containers.

setup/INFRA_VERSION

@@ -7,11 +7,11 @@ INFRA_VERSION=8
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.3.1
+BASE_IMAGE=cloudron/base:0.3.3
-MYSQL_IMAGE=cloudron/mysql:0.3.2
+MYSQL_IMAGE=cloudron/mysql:0.3.3
-POSTGRESQL_IMAGE=cloudron/postgresql:0.3.1
+POSTGRESQL_IMAGE=cloudron/postgresql:0.3.2
-MONGODB_IMAGE=cloudron/mongodb:0.3.1
+MONGODB_IMAGE=cloudron/mongodb:0.3.2
-REDIS_IMAGE=cloudron/redis:0.3.1 # if you change this, fix src/addons.js as well
+REDIS_IMAGE=cloudron/redis:0.3.2 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.3.1
+MAIL_IMAGE=cloudron/mail:0.3.2
-GRAPHITE_IMAGE=cloudron/graphite:0.3.3
+GRAPHITE_IMAGE=cloudron/graphite:0.3.4
 

setup/argparser.sh

@@ -16,6 +16,8 @@ arg_tls_key=""
 arg_token=""
 arg_version=""
 arg_web_server_origin=""
+arg_backup_key=""
+arg_aws=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -41,6 +43,12 @@ EOF
         arg_restore_key=$(echo "$2" | $json restoreKey)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
+        arg_backup_key=$(echo "$2" | $json backupKey)
+        [[ "${arg_backup_key}" == "null" ]] && arg_backup_key=""
+
+        arg_aws=$(echo "$2" | $json aws)
+        [[ "${arg_aws}" == "null" ]] && arg_aws=""
+
         shift 2
         ;;
     --) break;;

setup/container.sh

@@ -13,13 +13,10 @@ readonly DATA_DIR="/home/yellowtent/data"
 rm -rf "${CONFIG_DIR}"
 sudo -u yellowtent mkdir "${CONFIG_DIR}"
 
-########## logrotate (default ubuntu runs this daily)
+########## systemd
-rm -rf /etc/logrotate.d/*
+cp -r "${container_files}/systemd/." /etc/systemd/system/
-cp -r "${container_files}/logrotate/." /etc/logrotate.d/
+systemctl daemon-reload
-
+systemctl enable cloudron.target
-########## supervisor
-rm -rf /etc/supervisor/*
-cp -r "${container_files}/supervisor/." /etc/supervisor/
 
 ########## sudoers
 rm /etc/sudoers.d/*

setup/container/logrotate/cloudron

@@ -1,6 +0,0 @@
-/var/log/cloudron/*log {
-    missingok
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/logrotate/supervisor

@@ -1,7 +0,0 @@
-/var/log/supervisor/*log {
-    missingok
-    copytruncate
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/supervisor/conf.d/apphealthtask.conf

@@ -1,10 +0,0 @@
-[program:apphealthtask]
-command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/box.conf

@@ -1,10 +0,0 @@
-[program:box]
-command=/usr/bin/node "/home/yellowtent/box/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/crashnotifier.conf

@@ -1,11 +0,0 @@
-[eventlistener:crashnotifier]
-command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
-events=PROCESS_STATE
-autostart=true
-autorestart=true
-redirect_stderr=false
-stderr_logfile=/var/log/supervisor/crashnotifier.log
-stderr_logfile_maxbytes=50MB
-stderr_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/janitor.conf

@@ -1,10 +0,0 @@
-[program:janitor]
-command=/usr/bin/node "/home/yellowtent/box/janitor.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/janitor.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/oauthproxy.conf

@@ -1,10 +0,0 @@
-[program:oauthproxy]
-command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/oauthproxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/supervisord.conf

@@ -1,33 +0,0 @@
-; supervisor config file
-
-; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[supervisord]
-logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
-pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-nodaemon = false
-childlogdir = /var/log/supervisor/
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=http://127.0.0.1:9001
-
-; The [include] section can just contain the "files" setting.  This
-; setting can list multiple files (separated by whitespace or
-; newlines).  It can also contain wildcards.  The filenames are
-; interpreted as relative to this file.  Included files *cannot*
-; include files themselves.
-
-[include]
-files = conf.d/*.conf
-

setup/container/systemd/apphealthtask.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Cloudron App Health Monitor
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/apphealthtask.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/container/systemd/box.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/app.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smart Cloud
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=apphealthtask.service box.service janitor.service oauthproxy.service
+After=apphealthtask.service box.service janitor.service oauthproxy.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+

setup/container/systemd/janitor.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Cloudron Janitor
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/janitor.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M
+

setup/container/systemd/oauthproxy.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Cloudron OAuth Proxy Service
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/oauthproxy.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M
+

setup/start.sh

@@ -122,6 +122,7 @@ set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
 echo "Creating cloudron.conf"
+# note that arg_aws is a javascript object and intentionally unquoted below
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
@@ -138,7 +139,9 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "password": "${mysql_root_password}",
         "port": 3306,
         "name": "box"
-    }
+    },
+    "backupKey": "${arg_backup_key}",
+    "aws": ${arg_aws}
 }
 CONF_END
 
@@ -163,22 +166,10 @@ ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
 mysql -u root -p${mysql_root_password} \
     -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "80" "Reloading supervisor"
+set_progress "80" "Starting Cloudron"
-# looks like restarting supervisor completely is the only way to reload it
+systemctl start cloudron.target
-service supervisor stop || true
 
-echo -n "Waiting for supervisord to stop"
+sleep 2 # give systemd sometime to start the processes
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
 
 set_progress "85" "Reloading nginx"
 nginx -s reload

setup/stop.sh

@@ -2,14 +2,6 @@
 
 set -eu -o pipefail
 
-echo "Stopping box code"
+echo "Stopping cloudron"
-
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
 
+systemctl stop cloudron.target

src/addons.js

@@ -38,8 +38,7 @@ var appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
     util = require('util'),
     uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
+    vbox = require('./vbox.js');
-    _ = require('underscore');
 
 var NOOP = function (app, callback) { return callback(); };
 
@@ -665,7 +664,7 @@ function setupRedis(app, callback) {
         name: 'redis-' + app.id,
         Hostname: config.appFqdn(app.location),
         Tty: true,
-        Image: 'cloudron/redis:0.3.1',
+        Image: 'cloudron/redis:0.3.2', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
         Volumes: {},
         VolumesFrom: []

src/appdb.js

@@ -35,12 +35,13 @@ exports = module.exports = {
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
-    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
     RSTATE_STOPPED: 'stopped', // app stopped by use
+    RSTATE_ERROR: 'error',
 
+    // run codes (keep in sync in UI)
     HEALTH_HEALTHY: 'healthy',
     HEALTH_UNHEALTHY: 'unhealthy',
     HEALTH_ERROR: 'error',
@@ -335,6 +336,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
 
     // Rules are:
     // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
     // update and configure are allowed only in installed state
 

src/apps.js

@@ -490,28 +490,30 @@ function restore(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var restoreConfig = app.lastBackupConfig;
+        // restore without a backup is the same as re-install
-        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+        var restoreConfig = app.lastBackupConfig, values = { };
+        if (restoreConfig) {
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
-        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
+            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
-        error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(error);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
-        error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
+            // ## should probably query new location, access restriction from user
-        if (error) return callback(error);
+            values = {
+                manifest: restoreConfig.manifest,
+                portBindings: restoreConfig.portBindings,
 
-        // ## should probably query new location, access restriction from user
+                oldConfig: {
-        var values = {
+                    location: app.location,
-            manifest: restoreConfig.manifest,
+                    accessRestriction: app.accessRestriction,
-            portBindings: restoreConfig.portBindings,
+                    portBindings: app.portBindings,
-
+                    manifest: app.manifest
-            oldConfig: {
+                }
-                location: app.location,
+            };
-                accessRestriction: app.accessRestriction,
+        }
-                portBindings: app.portBindings,
-                manifest: app.manifest
-            }
-        };
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -573,6 +575,8 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
         return new Error('Box version exceeds Apps maxBoxVersion');
     }
@@ -664,7 +668,7 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
                 return iteratorDone();
             }
 
-           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
                 if (error) debug('Error initiating autoupdate of %s', appId);
 
                 iteratorDone(null);
@@ -700,7 +704,7 @@ function backupApp(app, addonsToBackup, callback) {
         return callback(safe.error);
     }
 
-    backups.getBackupUrl(app, null, function (error, result) {
+    backups.getBackupUrl(app, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -709,7 +713,7 @@ function backupApp(app, addonsToBackup, callback) {
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
             addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -756,7 +760,7 @@ function restoreApp(app, addonsToRestore, callback) {
 
         debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
 
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             addons.restoreAddons(app, addonsToRestore, callback);

src/apptask.js

@@ -46,6 +46,8 @@ var addons = require('./addons.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    SubdomainError = require('./subdomainerror.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
@@ -248,7 +250,7 @@ function deleteImage(app, manifest, callback) {
             noprune: false
         };
 
-        // delete image by id because docker pull pulls down all the tags and this is the only way to delete all tags
+         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
         docker.getImage(result.Id).remove(removeOptions, function (error) {
             if (error && error.statusCode === 404) return callback(null);
             if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -331,8 +333,12 @@ function startContainer(app, callback) {
             vbox.forwardFromHostToVirtualBox(app.id + '-tcp' + containerPort, hostPort);
         }
 
+        var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+
         var startOptions = {
             Binds: addons.getBindsSync(app, app.manifest.addons),
+            Memory: memoryLimit / 2,
+            MemorySwap: memoryLimit, // Memory + Swap
             PortBindings: dockerPortBindings,
             PublishAllPorts: false,
             Links: addons.getLinksSync(app, app.manifest.addons),
@@ -355,6 +361,11 @@ function startContainer(app, callback) {
 }
 
 function stopContainer(app, callback) {
+    if (!app.containerId) {
+        debugApp(app, 'No previous container to stop');
+        return callback();
+    }
+
     var container = docker.getContainer(app.containerId);
     debugApp(app, 'Stopping container %s', container.id);
 
@@ -414,49 +425,47 @@ function downloadIcon(app, callback) {
 }
 
 function registerSubdomain(app, callback) {
-    debugApp(app, 'Registering subdomain location [%s]', app.location);
-
     // even though the bare domain is already registered in the appstore, we still
     // need to register it so that we have a dnsRecordId to wait for it to complete
     var record = { subdomain: app.location, type: 'A', value: sysinfo.getIp() };
 
-    superagent
+    async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
+        debugApp(app, 'Registering subdomain location [%s]', app.location);
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: [ record ] })
-        .end(function (error, res) {
-            if (error) return callback(error);
 
-            debugApp(app, 'Registered subdomain status: %s', res.status);
+        subdomains.add(record, function (error, changeId) {
+            if (error && error.reason === SubdomainError.STILL_BUSY) return retryCallback(error); // try again
 
-            if (res.status === 409) return callback(null); // already registered
+            retryCallback(null, error || changeId);
-            if (res.status !== 201) return callback(new Error(util.format('Subdomain Registration failed. %s %j', res.status, res.body)));
-
-            updateApp(app, { dnsRecordId: res.body.ids[0] }, callback);
         });
+    }, function (error, result) {
+        if (error || result instanceof Error) return callback(error || result);
+
+        updateApp(app, { dnsRecordId: result }, callback);
+    });
 }
 
-function unregisterSubdomain(app, callback) {
+function unregisterSubdomain(app, location, callback) {
-    debugApp(app, 'Unregistering subdomain: dnsRecordId=%s', app.dnsRecordId);
-
-    if (!app.dnsRecordId) return callback(null);
-
     // do not unregister bare domain because we show a error/cloudron info page there
-    if (app.location === '') return updateApp(app, { dnsRecordId: null }, callback);
+    if (location === '') {
+        debugApp(app, 'Skip unregister of empty subdomain');
+        return callback(null);
+    }
 
-    superagent
+    var record = { subdomain: location, type: 'A', value: sysinfo.getIp() };
-        .del(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId)
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) {
-                debugApp(app, 'Error making request: %s', error);
-            } else if (res.status !== 204) {
-                debugApp(app, 'Error unregistering subdomain:', res.status, res.body);
-            }
 
-            updateApp(app, { dnsRecordId: null }, callback);
+    async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Unregistering subdomain: %s', location);
+
+        subdomains.remove(record, function (error) {
+            if (error && error.reason === SubdomainError.STILL_BUSY) return retryCallback(error); // try again
+
+            retryCallback(null);
         });
+    }, function (error) {
+        if (error) debugApp(app, 'Error unregistering subdomain: %s', error);
+
+        updateApp(app, { dnsRecordId: null }, callback);
+    });
 }
 
 function removeIcon(app, callback) {
@@ -477,21 +486,15 @@ function waitForDnsPropagation(app, callback) {
         setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
     }
 
-    superagent
+    subdomains.status(app.dnsRecordId, function (error, result) {
-        .get(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId + '/status')
+        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
 
-            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, res.status);
+        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-            if (res.status !== 200) return retry(new Error(util.format('Error getting record status: %s %j', res.status, res.body)));
+        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-            if (res.body.status !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, res.body.status)));
+        callback(null);
-
+    });
-            callback(null);
-        });
 }
 
 // updates the app object and the database
@@ -530,9 +533,9 @@ function install(app, callback) {
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
         deleteVolume.bind(null, app),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
         removeOAuthProxyCredentials.bind(null, app),
-        removeIcon.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
         unconfigureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
@@ -617,7 +620,11 @@ function restore(app, callback) {
          // oldConfig can be null during upgrades
         addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
         deleteVolume.bind(null, app),
-        deleteImage.bind(null, app, app.manifest),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -671,16 +678,15 @@ function restore(app, callback) {
 
 // note that configure is called after an infra update as well
 function configure(app, callback) {
-    var locationChanged = app.oldConfig ? app.oldConfig.location !== app.location : true;
-
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         function (next) {
-            if (!locationChanged) return next();
+            // oldConfig can be null during an infra update
-            unregisterSubdomain(app, next);
+            if (!app.oldConfig || app.oldConfig.location === app.location) return next();
+            unregisterSubdomain(app, app.oldConfig.location, next);
         },
         removeOAuthProxyCredentials.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -691,14 +697,8 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
         allocateOAuthProxyCredentials.bind(null, app),
 
-        function (next) {
+        updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
-            if (!locationChanged) return next();
+        registerSubdomain.bind(null, app),
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
-                registerSubdomain.bind(null, app)
-            ], next);
-        },
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         updateApp.bind(null, app, { installationProgress: '50, Setting up addons' }),
@@ -712,14 +712,8 @@ function configure(app, callback) {
 
         runApp.bind(null, app),
 
-        function (next) {
+        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
-            if (!locationChanged) return next();
+        exports._waitForDnsPropagation.bind(null, app),
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
-                exports._waitForDnsPropagation.bind(null, app)
-            ], next);
-        },
 
         // done!
         function (callback) {
@@ -753,7 +747,11 @@ function update(app, callback) {
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, unusedAddons),
-        deleteImage.bind(null, app, app.manifest), // delete image even if did not change (see df158b111f)
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
 
         function (next) {
@@ -819,7 +817,7 @@ function uninstall(app, callback) {
         deleteImage.bind(null, app, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
 
         updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
         removeOAuthProxyCredentials.bind(null, app),

src/aws.js

@@ -0,0 +1,253 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl,
+
+    addSubdomain: addSubdomain,
+    delSubdomain: delSubdomain,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('./config.js'),
+    debug = require('debug')('box:aws'),
+    SubdomainError = require('./subdomainerror.js'),
+    superagent = require('superagent');
+
+function getAWSCredentials(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
+        superagent.post(url).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 201) return callback(new Error(result.text));
+            if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
+
+            return callback(null, {
+                accessKeyId: result.body.credentials.AccessKeyId,
+                secretAccessKey: result.body.credentials.SecretAccessKey,
+                sessionToken: result.body.credentials.SessionToken,
+                region: 'us-east-1'
+            });
+        });
+    } else {
+        if (!config.aws().accessKeyId || !config.aws().secretAccessKey) return callback(new SubdomainError(SubdomainError.MISSING_CREDENTIALS));
+
+        callback(null, {
+            accessKeyId: config.aws().accessKeyId,
+            secretAccessKey: config.aws().secretAccessKey,
+            region: 'us-east-1'
+        });
+    }
+}
+
+function getSignedUploadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedUploadUrl: %s', filename);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedDownloadUrl: %s', filename);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getZoneByName(zoneName, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getZoneByName: %s', zoneName);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.listHostedZones({}, function (error, result) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+
+            var zone = result.HostedZones.filter(function (zone) {
+                return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+            })[0];
+
+            if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
+
+            debug('getZoneByName: found zone', zone);
+
+            callback(null, zone);
+        });
+    });
+}
+
+function addSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addSubdomain: ' + subdomain + ' for domain ' + zoneName + ' with value ' + value);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'UPSERT',
+                    ResourceRecordSet: {
+                        Type: type,
+                        Name: fqdn,
+                        ResourceRecords: [{
+                            Value: value
+                        }],
+                        Weight: 0,
+                        SetIdentifier: fqdn,
+                        TTL: 1
+                    }
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.code === 'PriorRequestNotComplete') {
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                } else if (error) {
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                }
+
+                debug('addSubdomain: success. changeInfoId:%j', result);
+
+                callback(null, result.ChangeInfo.Id);
+            });
+        });
+    });
+}
+
+function delSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('delSubdomain: %s for domain %s.', subdomain, zoneName);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var resourceRecordSet = {
+            Name: fqdn,
+            Type: type,
+            ResourceRecords: [{
+                Value: value
+            }],
+            Weight: 0,
+            SetIdentifier: fqdn,
+            TTL: 1
+        };
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'DELETE',
+                    ResourceRecordSet: resourceRecordSet
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                    debug('delSubdomain: resource record set not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'NoSuchHostedZone') {
+                    debug('delSubdomain: hosted zone not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'PriorRequestNotComplete') {
+                    debug('delSubdomain: resource is still busy', error);
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                } else if (error && error.code === 'InvalidChangeBatch') {
+                    debug('delSubdomain: invalid change batch. No such record to be deleted.');
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error) {
+                    debug('delSubdomain: error', error);
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                }
+
+                debug('delSubdomain: success');
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function getChangeStatus(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.getChange({ Id: changeId }, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, result.ChangeInfo.Status);
+        });
+    });
+}

src/backups.js

@@ -10,6 +10,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    aws = require('./aws.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
     superagent = require('superagent'),
@@ -54,42 +55,50 @@ function getAllPaged(page, perPage, callback) {
     });
 }
 
-function getBackupUrl(app, appBackupIds, callback) {
+function getBackupUrl(app, callback) {
     assert(!app || typeof app === 'object');
-    assert(!appBackupIds || util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+    var filename = '';
+    if (app) {
+        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    } else {
+        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
+    }
 
-    var data = {
+    aws.getSignedUploadUrl(filename, function (error, result) {
-        boxVersion: config.version(),
+        if (error) return callback(error);
-        appId: app ? app.id : null,
-        appVersion: app ? app.manifest.version : null,
-        appBackupIds: appBackupIds
-    };
 
-    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
+        var obj = {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+            id: filename,
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+            url: result.url,
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getBackupUrl: ', obj);
+
+        callback(null, obj);
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+    aws.getSignedDownloadUrl(backupId, function (error, result) {
+        if (error) return callback(error);
 
-    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
+        var obj = {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+            id: backupId,
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+            url: result.url,
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getRestoreUrl: ', obj);
+
+        callback(null, obj);
     });
 }
-
-

src/cloudron.js

@@ -39,6 +39,7 @@ var apps = require('./apps.js'),
     settings = require('./settings.js'),
     SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
@@ -46,7 +47,8 @@ var apps = require('./apps.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
 
 var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
@@ -54,7 +56,7 @@ var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
 
-var gAddMailDnsRecordsTimerId = null,
+var gAddDnsRecordsTimerId = null,
     gCloudronDetails = null;            // cached cloudron details like region,size...
 
 function debugApp(app, args) {
@@ -108,20 +110,17 @@ function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     if (process.env.BOX_ENV !== 'test') {
-        addMailDnsRecords();
+        addDnsRecords();
     }
 
-    // Send heartbeat once we are up and running, this speeds up the Cloudron creation, as otherwise we are bound to the cron.js settings
-    sendHeartbeat();
-
     callback(null);
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clearTimeout(gAddMailDnsRecordsTimerId);
+    clearTimeout(gAddDnsRecordsTimerId);
-    gAddMailDnsRecordsTimerId = null;
+    gAddDnsRecordsTimerId = null;
 
     callback(null);
 }
@@ -269,18 +268,20 @@ function getConfig(callback) {
 }
 
 function sendHeartbeat() {
+    // Only send heartbeats after the admin dns record is synced to give appstore a chance to know that fact
+    if (!config.get('dnsInSync')) return;
+
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
 
-    // TODO: this must be a POST
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
-    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
         if (error) debug('Error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
         else debug('Heartbeat sent to %s', url);
     });
 }
 
-function sendMailDnsRecordsRequest(callback) {
+function addDnsRecords() {
-    assert.strictEqual(typeof callback, 'function');
+    if (config.get('dnsInSync')) return sendHeartbeat(); // already registered send heartbeat
 
     var DKIM_SELECTOR = 'mail';
     var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
@@ -288,13 +289,20 @@ function sendMailDnsRecordsRequest(callback) {
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
 
-    if (publicKey === null) return callback(new Error('Error reading dkim public key'));
+    if (publicKey === null) {
+        console.error('Error reading dkim public key. Stop DNS setup.');
+        return;
+    }
 
     // remove header, footer and new lines
     publicKey = publicKey.split('\n').slice(1, -2).join('');
 
     // note that dmarc requires special DNS records for external RUF and RUA
     var records = [
+        // naked domain
+        { subdomain: '', type: 'A', value: sysinfo.getIp() },
+        // webadmin domain
+        { subdomain: 'my', type: 'A', value: sysinfo.getIp() },
         // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
         { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
         // t=s limits the domainkey to this domain and not it's subdomains
@@ -303,38 +311,47 @@ function sendMailDnsRecordsRequest(callback) {
         { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
     ];
 
-    debug('sendMailDnsRecords request:%s', JSON.stringify(records));
+    debug('addDnsRecords:', records);
 
-    superagent
+    subdomains.addMany(records, function (error, changeIds) {
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: records })
-        .end(function (error, res) {
-            if (error) return callback(error);
-
-            debug('sendMailDnsRecords status: %s', res.status);
-
-            if (res.status === 409) return callback(null); // already registered
-
-            if (res.status !== 201) return callback(new Error(util.format('Failed to add Mail DNS records: %s %j', res.status, res.body)));
-
-            return callback(null, res.body.ids);
-        });
-}
-
-function addMailDnsRecords() {
-    if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
-
-    sendMailDnsRecordsRequest(function (error, ids) {
         if (error) {
-            console.error('Mail DNS record addition failed', error);
+            console.error('Admin DNS record addition failed', error);
-            gAddMailDnsRecordsTimerId = setTimeout(addMailDnsRecords, 30000);
+            gAddDnsRecordsTimerId = setTimeout(addDnsRecords, 10000);
             return;
         }
 
-        debug('Added Mail DNS records successfully');
+        function checkIfInSync() {
-        config.set('mailDnsRecordIds', ids);
+            debug('addDnsRecords: Check if admin DNS record is in sync.');
+
+            var allDone = true;
+
+            async.each(changeIds, function (changeId, callback) {
+                subdomains.status(changeId, function (error, result) {
+                    if (error) return callback(new Error('Failed to check if admin DNS record is in sync.', error));
+
+                    if (result !== 'done') allDone = false;
+
+                    callback(null);
+                });
+            }, function (error) {
+                if (error) console.error(error);
+
+                // retry if needed
+                if (error || !allDone) {
+                    gAddDnsRecordsTimerId = setTimeout(checkIfInSync, 5000);
+                    return;
+                }
+
+                config.set('dnsInSync', true);
+
+                // send heartbeat after the dns records are done
+                sendHeartbeat();
+
+                debug('addDnsRecords: done');
+            });
+        }
+
+        checkIfInSync();
     });
 }
 
@@ -492,17 +509,19 @@ function doUpdate(boxUpdateInfo, callback) {
             var args = {
                 sourceTarballUrl: result.body.url,
 
-                // IMPORTANT: if you change this, fix up argparser.sh as well. keep these sorted for readability
+                // this data is opaque to the installer
                 data: {
                     apiServerOrigin: config.apiServerOrigin(),
+                    aws: config.aws(),
+                    backupKey: config.backupKey(),
                     boxVersionsUrl: config.get('boxVersionsUrl'),
                     fqdn: config.fqdn(),
                     isCustomDomain: config.isCustomDomain(),
-                    restoreKey: null,
                     restoreUrl: null,
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    restoreKey: null,
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
                     token: config.token(),
+                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
                     version: boxUpdateInfo.version,
                     webServerOrigin: config.webServerOrigin()
                 }
@@ -561,7 +580,7 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+    backups.getBackupUrl(null /* app */, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -569,14 +588,17 @@ function backupBoxWithAppBackupIds(appBackupIds, callback) {
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             debug('backup: successful');
 
-            callback(null, result.id);
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
+                callback(null, result.id);
+            });
         });
     });
 }
@@ -609,7 +631,7 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, 'Backing up app at ' + app.location);
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
 
                 if (error && error.reason === AppsError.BAD_STATE) {
                     debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);

src/config.js

@@ -30,6 +30,9 @@ exports = module.exports = {
 
     isDev: isDev,
 
+    backupKey: backupKey,
+    aws: aws,
+
     // for testing resets to defaults
     _reset: initConfig
 };
@@ -70,6 +73,14 @@ function initConfig() {
     data.webServerOrigin = null;
     data.internalPort = 3001;
     data.ldapPort = 3002;
+    data.backupKey = 'backupKey';
+    data.aws = {
+        backupBucket: null,
+        backupPrefix: null,
+        accessKeyId: null,      // selfhosting only
+        secretAccessKey: null   // selfhosting only
+    };
+    data.dnsInSync = false;
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -86,6 +97,8 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
+        data.aws.backupBucket = 'testbucket';
+        data.aws.backupPrefix = 'testprefix';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -99,6 +112,9 @@ function initConfig() {
     saveSync();
 }
 
+// cleanup any old config file we have for tests
+if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
+
 initConfig();
 
 // set(obj) or set(key, value)
@@ -172,3 +188,10 @@ function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
 
+function backupKey() {
+    return get('backupKey');
+}
+
+function aws() {
+    return get('aws');
+}

src/ldap.js

@@ -25,7 +25,7 @@ var gLogger = {
 };
 
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
-var GROUP_ADMINS_DN = 'cn=admin,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
 function start(callback) {
     assert(typeof callback === 'function');

src/locker.js

@@ -9,6 +9,7 @@ function Locker() {
     this._operation = null;
     this._timestamp = null;
     this._watcherId = -1;
+    this._lockDepth = 0; // recursive locks
 }
 util.inherits(Locker, EventEmitter);
 
@@ -24,6 +25,7 @@ Locker.prototype.lock = function (operation) {
     if (this._operation !== null) return new Error('Already locked for ' + this._operation);
 
     this._operation = operation;
+    ++this._lockDepth;
     this._timestamp = new Date();
     var that = this;
     this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
@@ -35,17 +37,31 @@ Locker.prototype.lock = function (operation) {
     return null;
 };
 
+Locker.prototype.recursiveLock = function (operation) {
+    if (this._operation === operation) {
+        ++this._lockDepth;
+        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
+        return null;
+    }
+
+    return this.lock(operation);
+};
+
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
-    debug('Released : %s', this._operation);
+    if (--this._lockDepth === 0) {
+        debug('Released : %s', this._operation);
 
-    this._operation = null;
+        this._operation = null;
-    this._timestamp = null;
+        this._timestamp = null;
-    clearInterval(this._watcherId);
+        clearInterval(this._watcherId);
-    this._watcherId = -1;
+        this._watcherId = -1;
+    } else {
+        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
+    }
 
     this.emit('unlocked', operation);
 

src/routes/oauth2.js

@@ -171,6 +171,8 @@ function sendErrorPageOrRedirect(req, res, message) {
     }
 }
 
+// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
+// This usually happens when the OAuth client ID is wrong
 function sendError(req, res, message) {
     assert.strictEqual(typeof req, 'object');
     assert.strictEqual(typeof res, 'object');

src/routes/test/backups-test.js

@@ -119,8 +119,8 @@ describe('Backups API', function () {
 
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
-                        .put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] })
+                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
-                        .reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                        .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             request.post(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })

src/routes/test/cloudron-test.js

@@ -450,8 +450,20 @@ describe('Cloudron', function () {
         });
 
         it('fails when in wrong state', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(409, {});
+            var scope2 = nock(config.apiServerOrigin())
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) { 
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
+                    return body.size && body.region && body.restoreKey;
+                }).reply(409, {});
 
             injectShellMock();
 
@@ -463,7 +475,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }
@@ -477,8 +489,19 @@ describe('Cloudron', function () {
 
 
         it('succeeds', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(202, {});
+            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                return body.size && body.region && body.restoreKey;
+            }).reply(202, {});
+
+            var scope2 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN', function (body) { 
+                        return body.boxVersion && body.restoreKey && !body.appId && !body.appVersion && body.appBackupIds.length === 0;
+                    })
+                    .reply(200, { id: 'someid' });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             injectShellMock();
 
@@ -490,7 +513,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }

src/scripts/backupapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: backup.sh <appid> <url> <key>"
+    echo "Usage: backupapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -22,6 +22,7 @@ readonly DATA_DIR="${HOME}/data"
 app_id="$1"
 backup_url="$2"
 backup_key="$3"
+session_token="$4"
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
@@ -31,9 +32,17 @@ btrfs subvolume snapshot -r "${app_data_dir}" "${app_data_snapshot}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${app_data_snapshot}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT "${headers[@]}" --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/backupbox.sh

@@ -13,12 +13,13 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 2 ]; then
-    echo "Usage: backupbox.sh <url> <key>"
+    echo "Usage: backupbox.sh <url> <key> [aws session token]"
     exit 1
 fi
 
 backup_url="$1"
 backup_key="$2"
+session_token="$3"
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"
@@ -32,9 +33,17 @@ btrfs subvolume snapshot -r "${BOX_DATA_DIR}" "${box_snapshot_dir}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${box_snapshot_dir}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/collectlogs.sh

@@ -21,7 +21,7 @@ readonly program_name=$1
 
 echo "${program_name}.log"
 echo "-------------------"
-tail --lines=100 /var/log/supervisor/${program_name}.log
+journalctl --no-pager -u ${program_name} -n 100
 echo
 echo
 echo "dmesg"
@@ -31,7 +31,7 @@ echo
 echo
 echo "docker"
 echo "------"
-tail --lines=100 /var/log/upstart/docker.log
+journalctl --no-pager -u docker -n 50
 echo
 echo
 

src/scripts/reloadnginx.sh

@@ -12,12 +12,6 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${OSTYPE}" == "darwin"* ]]; then
-    # On Mac, brew installs supervisor in /usr/local/bin
-    export PATH=$PATH:/usr/local/bin
-fi
-
 if [[ "${BOX_ENV}" == "cloudron" ]]; then
     nginx -s reload
 fi
-

src/scripts/restoreapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: restoreapp.sh <appid> <url> <key>"
+    echo "Usage: restoreapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -23,6 +23,7 @@ readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max
 app_id="$1"
 restore_url="$2"
 restore_key="$3"
+session_token="$4"
 
 echo "Downloading backup: ${restore_url} and key: ${restore_key}"
 
@@ -30,7 +31,14 @@ for try in `seq 1 5`; do
     echo "Download backup from ${restore_url} (try ${try})"
     error_log=$(mktemp)
 
-    if $curl -L "${restore_url}" \
+    headers=("") # empty element required (http://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u)
+
+    # federated tokens in CaaS case need session token
+    if [[ ! -z "${session_token}" ]]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
+    if $curl -L "${headers[@]}" "${restore_url}" \
         | openssl aes-256-cbc -d -pass "pass:${restore_key}" \
         | tar -zxf - -C "${DATA_DIR}/${app_id}" 2>"${error_log}"; then
         chown -R yellowtent:yellowtent "${DATA_DIR}/${app_id}"

src/server.js

@@ -97,7 +97,7 @@ function initializeExpressSync() {
     // private routes
     router.get ('/api/v1/cloudron/config', rootScope, routes.cloudron.getConfig);
     router.post('/api/v1/cloudron/update', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
-    router.get ('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
+    router.post('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
     router.post('/api/v1/cloudron/migrate', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.migrate);
     router.post('/api/v1/cloudron/certificate', rootScope, multipart, routes.cloudron.setCertificate);
     router.get ('/api/v1/cloudron/graphs', rootScope, routes.graphs.getGraphs);

src/subdomainerror.js

@@ -0,0 +1,33 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    util = require('util');
+
+exports = module.exports = SubdomainError;
+
+function SubdomainError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(SubdomainError, Error);
+
+SubdomainError.NOT_FOUND = 'No such domain';
+SubdomainError.EXTERNAL_ERROR = 'External error';
+SubdomainError.STILL_BUSY = 'Still busy';
+SubdomainError.MISSING_CREDENTIALS = 'Missing credentials';

src/subdomains.js

@@ -0,0 +1,82 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    aws = require('./aws.js'),
+    config = require('./config.js'),
+    debug = require('debug')('box:subdomains'),
+    util = require('util'),
+    SubdomainError = require('./subdomainerror.js');
+
+module.exports = exports = {
+    add: add,
+    addMany: addMany,
+    remove: remove,
+    status: status
+};
+
+function add(record, callback) {
+    assert.strictEqual(typeof record, 'object');
+    assert.strictEqual(typeof record.subdomain, 'string');
+    assert.strictEqual(typeof record.type, 'string');
+    assert.strictEqual(typeof record.value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: ', record);
+
+    aws.addSubdomain(config.zoneName(), record.subdomain, record.type, record.value, function (error, changeId) {
+        if (error) return callback(error);
+        callback(null, changeId);
+    });
+}
+
+function addMany(records, callback) {
+    assert(util.isArray(records));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addMany: ', records);
+
+    var changeIds = [];
+
+    async.eachSeries(records, function (record, callback) {
+        add(record, function (error, changeId) {
+            if (error) return callback(error);
+
+            changeIds.push(changeId);
+
+            callback(null);
+        });
+    }, function (error) {
+        if (error) return callback(error);
+        callback(null, changeIds);
+    });
+}
+
+function remove(record, callback) {
+    assert.strictEqual(typeof record, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('remove: ', record);
+
+    aws.delSubdomain(config.zoneName(), record.subdomain, record.type, record.value, function (error) {
+        if (error && error.reason !== SubdomainError.NOT_FOUND) return callback(error);
+
+        debug('deleteSubdomain: successfully deleted subdomain from aws.');
+
+        callback(null);
+    });
+}
+
+function status(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('status: ', changeId);
+
+    aws.getChangeStatus(changeId, function (error, status) {
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error));
+        callback(null, status === 'INSYNC' ? 'done' : 'pending');
+    });
+}

src/taskmanager.js

@@ -17,10 +17,7 @@ var appdb = require('./appdb.js'),
 var gActiveTasks = { };
 var gPendingTasks = [ ];
 
-// Task concurrency is 1 for two reasons:
+var TASK_CONCURRENCY = 5;
-// 1. The backup scripts (app and box) turn off swap after finish disregarding other backup processes
-// 2. apptask getFreePort has race with multiprocess
-var TASK_CONCURRENCY = 1;
 var NOOP_CALLBACK = function (error) { console.error(error); };
 
 function initialize(callback) {
@@ -31,6 +28,8 @@ function initialize(callback) {
         if (error) return callback(error);
 
         apps.forEach(function (app) {
+            if (app.installationState === appdb.ISTATE_INSTALLED && app.runState === appdb.RSTATE_RUNNING) return;
+
             debug('Creating process for %s (%s) with state %s', app.location, app.id, app.installationState);
             startAppTask(app.id);
         });
@@ -54,7 +53,8 @@ function uninitialize(callback) {
 
 function startNextTask() {
     if (gPendingTasks.length === 0) return;
-    assert.strictEqual(Object.keys(gActiveTasks).length, 0); // since we allow only one task at a time
+
+    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     startAppTask(gPendingTasks.shift());
 }
@@ -63,14 +63,20 @@ function startAppTask(appId) {
     assert.strictEqual(typeof appId, 'string');
     assert(!(appId in gActiveTasks));
 
-    var lockError = locker.lock(locker.OP_APPTASK);
+    if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
-
-    if (lockError || Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug('Reached concurrency limit, queueing task for %s', appId);
         gPendingTasks.push(appId);
         return;
     }
 
+    var lockError = locker.recursiveLock(locker.OP_APPTASK);
+
+    if (lockError) {
+        debug('Locked for another operation, queueing task for %s', appId);
+        gPendingTasks.push(appId);
+        return;
+    }
+
     gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
     gActiveTasks[appId].once('exit', function (code) {
         debug('Task for %s completed with status %s', appId, code);

src/test/apptask-test.js

@@ -1,5 +1,6 @@
 /* jslint node:true */
 /* global it:false */
+/* global xit:false */
 /* global describe:false */
 /* global before:false */
 /* global after:false */
@@ -154,7 +155,7 @@ describe('apptask', function () {
     it('barfs on bad manifest', function (done) {
         var badApp = _.extend({ }, APP);
         badApp.manifest = _.extend({ }, APP.manifest);
-        delete badApp.manifest['id'];
+        delete badApp.manifest.id;
 
         apptask._verifyManifest(badApp, function (error) {
             expect(error).to.be.ok();
@@ -182,9 +183,11 @@ describe('apptask', function () {
         });
     });
 
-    it('registers subdomain', function (done) {
+    xit('registers subdomain', function (done) {
         nock.cleanAll();
         var scope = nock(config.apiServerOrigin())
+            .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+            .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } })
             .post('/api/v1/subdomains?token=' + config.token(), { records: [ { subdomain: APP.location, type: 'A', value: sysinfo.getIp() } ] })
             .reply(201, { ids: [ APP.dnsRecordId ] });
 
@@ -195,7 +198,7 @@ describe('apptask', function () {
         });
     });
 
-    it('unregisters subdomain', function (done) {
+    xit('unregisters subdomain', function (done) {
         nock.cleanAll();
         var scope = nock(config.apiServerOrigin())
             .delete('/api/v1/subdomains/' + APP.dnsRecordId + '?token=' + config.token())

src/updatechecker.js

@@ -10,21 +10,15 @@ exports = module.exports = {
 };
 
 var apps = require('./apps.js'),
-    assert = require('assert'),
     async = require('async'),
     config = require('./config.js'),
     debug = require('debug')('box:updatechecker'),
-    fs = require('fs'),
     mailer = require('./mailer.js'),
-    path = require('path'),
-    paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     superagent = require('superagent'),
     util = require('util');
 
-var NOOP_CALLBACK = function (error) { console.error(error); };
-
 var gAppUpdateInfo = { }, // id -> update info { creationDate, manifest }
     gBoxUpdateInfo = null,
     gMailedUser =  { };
@@ -138,6 +132,7 @@ function checkAppUpdates() {
 
                 mailer.appUpdateAvailable(app, gAppUpdateInfo[id]);
                 gMailedUser[id] = true;
+                iteratorDone();
             });
         });
     });
@@ -157,4 +152,3 @@ function checkBoxUpdates() {
         }
     });
 }
-

src/user.js

@@ -203,17 +203,17 @@ function verifyWithEmail(email, password, callback) {
     });
 }
 
-function removeUser(username, callback) {
+function removeUser(userId, callback) {
-    assert.strictEqual(typeof username, 'string');
+    assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.del(username, function (error) {
+    userdb.del(userId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
         callback(null);
 
-        mailer.userRemoved(username);
+        mailer.userRemoved(userId);
     });
 }
 

src/webhooks.js

@@ -0,0 +1,47 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    backupDone: backupDone
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:webhooks'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function backupDone(filename, app, appBackupIds, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert(!app || typeof app === 'object');
+    assert(!appBackupIds || util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('backupDone():', filename);
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupDone';
+        var data = {
+            boxVersion: config.version(),
+            restoreKey: filename,
+            appId: app ? app.id : null,
+            appVersion: app ? app.manifest.version : null,
+            appBackupIds: appBackupIds
+        };
+
+        superagent.post(url).send(data).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 200) return callback(new Error(result.text));
+            if (!result.body) return callback(new Error('Unexpected response'));
+
+            debug('backupDone()', filename);
+
+            return callback(null);
+        });
+    } else {
+        // TODO call custom webhook
+        callback(null);
+    }
+}

webadmin/src/appstatus.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron App Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/error.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/index.html

@@ -8,6 +8,9 @@
 
     <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -48,9 +51,6 @@
     <!-- Main Application -->
     <script src="js/index.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body>
@@ -81,7 +81,7 @@
                         <li ng-repeat="change in config.update.box.changelog">{{change}}</li>
                     </ul>
                     <br/>
-                    <fieldset>
+                    <fieldset ng-show="installedApps | readyToUpdate">
                         <form name="update_form" role="form" ng-submit="doUpdate()" autocomplete="off">
                             <div class="form-group" ng-class="{ 'has-error': update_form.password.$dirty && update_form.password.$invalid }">
                                 <label class="control-label" for="inputUpdatePassword">Give your password to verify that you are performing that action</label>
@@ -99,7 +99,7 @@
                 </div>
                 <div class="modal-footer">
                     <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
+                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy" ng-show="installedApps | readyToUpdate"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
                 </div>
             </div>
         </div>

webadmin/src/js/client.js

@@ -425,7 +425,7 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
     };
 
     Client.prototype.reboot = function (callback) {
-        $http.get(client.apiOrigin + '/api/v1/cloudron/reboot').success(function(data, status) {
+        $http.post(client.apiOrigin + '/api/v1/cloudron/reboot', { }).success(function(data, status) {
             if (status !== 202 || typeof data !== 'object') return callback(new ClientError(status, data));
             callback(null, data);
         }).error(defaultErrorHandler(callback));

webadmin/src/js/index.js

@@ -96,13 +96,13 @@ app.filter('installationStateLabel', function() {
         var waiting = app.progress === 0 ? ' (Waiting)' : '';
 
         switch (app.installationState) {
-        case ISTATES.PENDING_INSTALL: return 'Installing...' + waiting;
+        case ISTATES.PENDING_INSTALL: return 'Installing' + waiting;
-        case ISTATES.PENDING_CONFIGURE: return 'Configuring...' + waiting;
+        case ISTATES.PENDING_CONFIGURE: return 'Configuring' + waiting;
-        case ISTATES.PENDING_UNINSTALL: return 'Uninstalling...' + waiting;
+        case ISTATES.PENDING_UNINSTALL: return 'Uninstalling' + waiting;
-        case ISTATES.PENDING_RESTORE: return 'Restoring...' + waiting;
+        case ISTATES.PENDING_RESTORE: return 'Restoring' + waiting;
-        case ISTATES.PENDING_UPDATE: return 'Updating...' + waiting;
+        case ISTATES.PENDING_UPDATE: return 'Updating' + waiting;
-        case ISTATES.PENDING_FORCE_UPDATE: return 'Updating...' + waiting;
+        case ISTATES.PENDING_FORCE_UPDATE: return 'Updating' + waiting;
-        case ISTATES.PENDING_BACKUP: return 'Backing up...' + waiting;
+        case ISTATES.PENDING_BACKUP: return 'Backing up' + waiting;
         case ISTATES.ERROR: return 'Error';
         case ISTATES.INSTALLED: {
             if (app.runState === 'running') {

webadmin/src/nakeddomain.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/setup.html

@@ -6,6 +6,9 @@
 
     <title> Cloudron Setup </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -31,9 +34,6 @@
     <!-- Setup Application -->
     <script src="js/setup.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body class="setup" ng-app="Application" ng-controller="SetupController">

webadmin/src/theme.scss

@@ -195,8 +195,8 @@ html {
 
 .appstore-item-badge-testing {
     position: absolute;
-    right: 15px;
+    right: 0;
-    top: 15px;
+    top: 2px;
 }
 
 .appstore-item-content-testing {
@@ -330,6 +330,10 @@ html {
     background-color: #5CB85C;
 }
 
+.badge-warning {
+    background-color: #EFBD48;
+}
+
 .badge-danger {
     background-color: $brand-danger;
 }

webadmin/src/update.html

@@ -6,6 +6,9 @@
 
     <title> Cloudron Update </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -23,9 +26,6 @@
     <!-- Update Application -->
     <script src="js/update.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body ng-app="Application" ng-controller="Controller" style="background-color: #7F7F7F">

webadmin/src/views/apps.html

@@ -269,3 +269,6 @@
         </div>
     </div>
 </div>
+
+<!-- Offset the footer -->
+<br/><br/>

webadmin/src/views/appstore.html

@@ -147,8 +147,9 @@
         <div class="col-md-10" ng-show="ready && apps.length">
             <div class="row-no-margin">
                 <div class="col-sm-1 appstore-item" ng-repeat="app in apps">
-                    <div class="appstore-item-content highlight" ng-click="showInstall(app)" ng-class="{ 'appstore-item-content-testing': app.publishState === 'testing' }">
+                    <div class="appstore-item-content highlight" ng-click="showInstall(app)" ng-class="{ 'appstore-item-content-testing': (app.publishState === 'testing' || app.publishState === 'pending_approval') }">
                         <span class="badge badge-danger appstore-item-badge-testing" ng-show="app.publishState === 'testing'">Testing</span>
+                        <span class="badge badge-warning appstore-item-badge-testing" ng-show="app.publishState === 'pending_approval'">Pending Approval</span>
                         <div class="appstore-item-content-icon col-same-height">
                             <img ng-src="{{app.iconUrl}}" onerror="this.onerror=null;this.src='img/appicon_fallback.png'" class="app-icon"/>
                         </div>

webadmin/src/views/graphs.html

@@ -14,39 +14,15 @@
     <div class="row shadow memory-app-container">
         <h2>Disk Usage</h2>
         <br/>
-        <div class="col-md-4">
+        <div class="col-md-offset-4 col-md-4">
-            <h4>Applications <span class="badge">{{ diskUsage['docker'].sum }} GB</span></h4>
-            <canvas id="dockerDiskUsageChart" width="200" height="200"></canvas>
-            <p>
-                <span class="text-success">Free {{ diskUsage['docker'].free }} GB</span>
-                &nbsp;
-                <span class="text-warning">Reserved {{ diskUsage['docker'].reserved }} GB</span>
-                &nbsp;
-                <span class="text-primary">Used {{ diskUsage['docker'].used }} GB</span>
-            </p>
-        </div>
-        <div class="col-md-4">
             <h4>Data <span class="badge">{{ diskUsage['box'].sum }} GB</span></h4>
             <canvas id="boxDiskUsageChart" width="200" height="200"></canvas>
             <p>
                 <span class="text-success">Free {{ diskUsage['box'].free }} GB</span>
                 &nbsp;
-                <span class="text-warning">Reserved {{ diskUsage['box'].reserved }} GB</span>
-                &nbsp;
                 <span class="text-primary">Used {{ diskUsage['box'].used }} GB</span>
             </p>
         </div>
-        <div class="col-md-4">
-            <h4>System (all) <span class="badge">{{ diskUsage['cloudron'].sum }} GB</span></h4>
-            <canvas id="cloudronDiskUsageChart" width="200" height="200"></canvas>
-            <p>
-                <span class="text-success">Free {{ diskUsage['cloudron'].free }} GB</span>
-                &nbsp;
-                <span class="text-warning">Reserved {{ diskUsage['cloudron'].reserved }} GB</span>
-                &nbsp;
-                <span class="text-primary">Used {{ diskUsage['cloudron'].used }} GB</span>
-            </p>
-        </div>
     </div>
 
     <br/>

webadmin/src/views/graphs.js

@@ -33,8 +33,7 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
 
     function renderDisk(type, free, reserved, used) {
         $scope.diskUsage[type] = {
-            used: bytesToGigaBytes(used.datapoints[0][0]),
+            used: bytesToGigaBytes(used.datapoints[0][0] + reserved.datapoints[0][0]),
-            reserved: bytesToGigaBytes(reserved.datapoints[0][0]),
             free: bytesToGigaBytes(free.datapoints[0][0]),
             sum: bytesToGigaBytes(used.datapoints[0][0] + reserved.datapoints[0][0] + free.datapoints[0][0])
         };
@@ -44,11 +43,6 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
             color: "#2196F3",
             highlight: "#82C4F8",
             label: "Used"
-        }, {
-            value: $scope.diskUsage[type].reserved,
-            color: "#f0ad4e",
-            highlight: "#F8D9AC",
-            label: "Reserved"
         }, {
             value: $scope.diskUsage[type].free,
             color:"#27CE65",
@@ -98,7 +92,7 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
             var options = {
                 scaleOverride: true,
                 scaleSteps: 10,
-                scaleStepWidth: $scope.activeApp === 'system' ? 100 : 10,
+                scaleStepWidth: $scope.activeApp === 'system' ? 200 : 60,
                 scaleStartValue: 0
             };
 
@@ -111,21 +105,11 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
         Client.graphs([
             'averageSeries(collectd.localhost.df-loop0.df_complex-free)',
             'averageSeries(collectd.localhost.df-loop0.df_complex-reserved)',
-            'averageSeries(collectd.localhost.df-loop0.df_complex-used)',
+            'averageSeries(collectd.localhost.df-loop0.df_complex-used)'
-
-            'averageSeries(collectd.localhost.df-loop1.df_complex-free)',
-            'averageSeries(collectd.localhost.df-loop1.df_complex-reserved)',
-            'averageSeries(collectd.localhost.df-loop1.df_complex-used)',
-
-            'averageSeries(collectd.localhost.df-vda1.df_complex-free)',
-            'averageSeries(collectd.localhost.df-vda1.df_complex-reserved)',
-            'averageSeries(collectd.localhost.df-vda1.df_complex-used)',
         ], '-1min', function (error, data) {
             if (error) return console.log(error);
 
-            renderDisk('docker', data[0], data[1], data[2]);
+            renderDisk('box', data[0], data[1], data[2]);
-            renderDisk('box', data[3], data[4], data[5]);
-            renderDisk('cloudron', data[6], data[7], data[8]);
         });
     };