systemd: fix crashnotifier

This removes logrotate as well since we use systemd logging

.gitignore

@@ -4,10 +4,6 @@ docs/
 webadmin/dist/
 setup/splash/website/
 
-# vim swam files
+# vim swap files
 *.swp
 
-# supervisor
-supervisord.pid
-supervisord.log
-

README.md

@@ -4,7 +4,6 @@ The Box
 Development setup
 -----------------
 * sudo useradd -m yellowtent
-** This dummy user is required for supervisor 'box' configs
 ** Add admin-localhost as 127.0.0.1 in /etc/hosts
 ** All apps will be installed as hypened-subdomains of localhost. You should add
    hyphened-subdomains of your apps into /etc/hosts

crashnotifier.js

@@ -2,77 +2,46 @@
 
 'use strict';
 
-// WARNING This is a supervisor eventlistener!
-//         The communication happens via stdin/stdout
-//         !! No console.log() allowed
-//         !! Do not set DEBUG
+var assert = require('assert'),
+    mailer = require('./src/mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
 
-var supervisor = require('supervisord-eventlistener'),
-    assert = require('assert'),
-    exec = require('child_process').exec,
-    util = require('util'),
-    fs = require('fs'),
-    mailer = require('./src/mailer.js');
-
-var gLastNotifyTime = {};
-var gCooldownTime = 1000 * 60  * 5; // 5 min
+var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
 
 function collectLogs(program, callback) {
     assert.strictEqual(typeof program, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var logFilePath = util.format('/var/log/supervisor/%s.log', program);
-
-    if (!fs.existsSync(logFilePath)) return callback(new Error(util.format('Log file %s does not exist.', logFilePath)));
-
-    fs.readFile(logFilePath, 'utf-8', function (error, data) {
-        if (error) return callback(error);
-
-        var lines = data.split('\n');
-        var boxLogLines = lines.slice(-100);
-
-        exec('dmesg', function (error, stdout /*, stderr */) {
-            if (error) console.error(error);
-
-            var lines = stdout.split('\n');
-            var dmesgLogLines = lines.slice(-100);
-
-            var result = '';
-            result += program + '.log\n';
-            result += '-------------------------------------\n';
-            result += boxLogLines.join('\n');
-            result += '\n\n';
-            result += 'dmesg\n';
-            result += '-------------------------------------\n';
-            result += dmesgLogLines.join('\n');
-
-            callback(null, result);
-        });
-    });
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
+    callback(null, logs);
 }
 
-supervisor.on('PROCESS_STATE_EXITED', function (headers, data) {
-    if (data.expected === '1') return console.error('Normal app %s exit', data.processname);
-
-    console.error('%s exited unexpectedly', data.processname);
-
-    collectLogs(data.processname, function (error, result) {
+function sendCrashNotification(processName) {
+    collectLogs(processName, function (error, result) {
         if (error) {
             console.error('Failed to collect logs.', error);
             result = util.format('Failed to collect logs.', error);
         }
 
-        if (!gLastNotifyTime[data.processname] || gLastNotifyTime[data.processname] < Date.now() - gCooldownTime) {
-            console.error('Send mail.');
-            mailer.sendCrashNotification(data.processname, result);
-            gLastNotifyTime[data.processname] = Date.now();
-        } else {
-            console.error('Do not send mail, already sent one recently.');
-        }
+        console.log('Sending crash notification email for', processName);
+        mailer.sendCrashNotification(processName, result);
     });
-});
+}
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    mailer.initialize(function (error) {
+        if (error) return console.error(error);
+
+        sendCrashNotification(processName);
+    });
+}
+
+main();
 
-mailer.initialize(function () {
-    supervisor.listen(process.stdin, process.stdout);
-    console.error('Crashnotifier listening...');
-});

npm-shrinkwrap.json

@@ -7,6 +7,28 @@
       "from": "https://registry.npmjs.org/async/-/async-1.2.1.tgz",
       "resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz"
     },
+    "aws-sdk": {
+      "version": "2.1.46",
+      "from": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "dependencies": {
+        "sax": {
+          "version": "0.5.3",
+          "from": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz",
+          "resolved": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz"
+        },
+        "xml2js": {
+          "version": "0.2.8",
+          "from": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz",
+          "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz"
+        },
+        "xmlbuilder": {
+          "version": "0.4.2",
+          "from": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz",
+          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz"
+        }
+      }
+    },
     "body-parser": {
       "version": "1.13.1",
       "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.13.1.tgz",
@@ -105,23 +127,24 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "1.4.0",
-      "from": "cloudron-manifestformat@1.4.0",
+      "version": "1.7.0",
+      "from": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
       "dependencies": {
         "java-packagename-regex": {
           "version": "1.0.0",
-          "from": "java-packagename-regex@>=1.0.0 <2.0.0",
+          "from": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz",
           "resolved": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz"
         },
         "safetydance": {
           "version": "0.0.15",
-          "from": "safetydance@0.0.15",
+          "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz",
           "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz"
         },
         "tv4": {
-          "version": "1.1.12",
-          "from": "tv4@>=1.1.9 <2.0.0",
-          "resolved": "http://registry.npmjs.org/tv4/-/tv4-1.1.12.tgz"
+          "version": "1.2.3",
+          "from": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz",
+          "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz"
         }
       }
     },
@@ -131,18 +154,17 @@
       "resolved": "https://registry.npmjs.org/connect-ensure-login/-/connect-ensure-login-0.1.1.tgz"
     },
     "connect-lastmile": {
-      "version": "0.0.12",
-      "from": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.12.tgz",
-      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.12.tgz",
+      "version": "0.0.13",
+      "from": "connect-lastmile@0.0.13",
       "dependencies": {
         "debug": {
           "version": "2.1.3",
-          "from": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
+          "from": "debug@>=2.1.0 <2.2.0",
           "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
           "dependencies": {
             "ms": {
               "version": "0.7.0",
-              "from": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz",
+              "from": "ms@0.7.0",
               "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
             }
           }
@@ -1288,69 +1310,69 @@
     },
     "dockerode": {
       "version": "2.2.2",
-      "from": "dockerode@2.2.2",
+      "from": "https://registry.npmjs.org/dockerode/-/dockerode-2.2.2.tgz",
       "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.2.2.tgz",
       "dependencies": {
         "docker-modem": {
           "version": "0.2.6",
-          "from": "docker-modem@>=0.2.0 <0.3.0",
+          "from": "https://registry.npmjs.org/docker-modem/-/docker-modem-0.2.6.tgz",
           "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-0.2.6.tgz",
           "dependencies": {
             "debug": {
               "version": "0.7.4",
-              "from": "debug@>=0.7.4 <0.8.0",
+              "from": "https://registry.npmjs.org/debug/-/debug-0.7.4.tgz",
               "resolved": "https://registry.npmjs.org/debug/-/debug-0.7.4.tgz"
             },
             "follow-redirects": {
               "version": "0.0.3",
-              "from": "follow-redirects@0.0.3",
+              "from": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.3.tgz",
               "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-0.0.3.tgz"
             },
             "JSONStream": {
               "version": "0.10.0",
-              "from": "JSONStream@0.10.0",
+              "from": "https://registry.npmjs.org/JSONStream/-/JSONStream-0.10.0.tgz",
               "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-0.10.0.tgz",
               "dependencies": {
                 "jsonparse": {
                   "version": "0.0.5",
-                  "from": "jsonparse@0.0.5",
+                  "from": "https://registry.npmjs.org/jsonparse/-/jsonparse-0.0.5.tgz",
                   "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-0.0.5.tgz"
                 },
                 "through": {
                   "version": "2.3.8",
-                  "from": "through@>=2.2.7 <3.0.0",
+                  "from": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
                   "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz"
                 }
               }
             },
             "querystring": {
               "version": "0.2.0",
-              "from": "querystring@0.2.0",
+              "from": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
               "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz"
             },
             "readable-stream": {
               "version": "1.0.33",
-              "from": "readable-stream@>=1.0.26-4 <1.1.0",
+              "from": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz",
               "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.33.tgz",
               "dependencies": {
                 "core-util-is": {
                   "version": "1.0.1",
-                  "from": "core-util-is@>=1.0.0 <1.1.0",
+                  "from": "http://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz",
                   "resolved": "http://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
                 },
                 "isarray": {
                   "version": "0.0.1",
-                  "from": "isarray@0.0.1",
+                  "from": "http://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
                   "resolved": "http://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
                 },
                 "string_decoder": {
                   "version": "0.10.31",
-                  "from": "string_decoder@>=0.10.0 <0.11.0",
+                  "from": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
                   "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
                 },
                 "inherits": {
                   "version": "2.0.1",
-                  "from": "inherits@>=2.0.1 <2.1.0",
+                  "from": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
                   "resolved": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
                 }
               }
@@ -1691,88 +1713,81 @@
     },
     "ldapjs": {
       "version": "0.7.1",
-      "from": "ldapjs@*",
+      "from": "https://registry.npmjs.org/ldapjs/-/ldapjs-0.7.1.tgz",
       "resolved": "https://registry.npmjs.org/ldapjs/-/ldapjs-0.7.1.tgz",
       "dependencies": {
         "asn1": {
           "version": "0.2.1",
-          "from": "asn1@0.2.1",
+          "from": "https://registry.npmjs.org/asn1/-/asn1-0.2.1.tgz",
           "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.1.tgz"
         },
         "assert-plus": {
           "version": "0.1.5",
-          "from": "assert-plus@0.1.5",
+          "from": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz",
           "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz"
         },
         "bunyan": {
           "version": "0.22.1",
-          "from": "bunyan@0.22.1",
-          "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.1.tgz",
-          "dependencies": {
-            "mv": {
-              "version": "0.0.5",
-              "from": "mv@0.0.5",
-              "resolved": "https://registry.npmjs.org/mv/-/mv-0.0.5.tgz"
-            }
-          }
+          "from": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.1.tgz",
+          "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-0.22.1.tgz"
         },
         "nopt": {
           "version": "2.1.1",
-          "from": "nopt@2.1.1",
+          "from": "https://registry.npmjs.org/nopt/-/nopt-2.1.1.tgz",
           "resolved": "https://registry.npmjs.org/nopt/-/nopt-2.1.1.tgz",
           "dependencies": {
             "abbrev": {
               "version": "1.0.7",
-              "from": "abbrev@>=1.0.0 <2.0.0",
+              "from": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.7.tgz",
               "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.7.tgz"
             }
           }
         },
         "pooling": {
           "version": "0.4.6",
-          "from": "pooling@0.4.6",
+          "from": "https://registry.npmjs.org/pooling/-/pooling-0.4.6.tgz",
           "resolved": "https://registry.npmjs.org/pooling/-/pooling-0.4.6.tgz",
           "dependencies": {
             "once": {
               "version": "1.3.0",
-              "from": "once@1.3.0",
+              "from": "https://registry.npmjs.org/once/-/once-1.3.0.tgz",
               "resolved": "https://registry.npmjs.org/once/-/once-1.3.0.tgz"
             },
             "vasync": {
               "version": "1.4.0",
-              "from": "vasync@1.4.0",
+              "from": "https://registry.npmjs.org/vasync/-/vasync-1.4.0.tgz",
               "resolved": "https://registry.npmjs.org/vasync/-/vasync-1.4.0.tgz",
               "dependencies": {
                 "jsprim": {
                   "version": "0.3.0",
-                  "from": "jsprim@0.3.0",
+                  "from": "https://registry.npmjs.org/jsprim/-/jsprim-0.3.0.tgz",
                   "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-0.3.0.tgz",
                   "dependencies": {
                     "extsprintf": {
                       "version": "1.0.0",
-                      "from": "extsprintf@1.0.0",
+                      "from": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.0.tgz",
                       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.0.tgz"
                     },
                     "json-schema": {
                       "version": "0.2.2",
-                      "from": "json-schema@0.2.2",
+                      "from": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz",
                       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz"
                     },
                     "verror": {
                       "version": "1.3.3",
-                      "from": "verror@1.3.3",
+                      "from": "https://registry.npmjs.org/verror/-/verror-1.3.3.tgz",
                       "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.3.tgz"
                     }
                   }
                 },
                 "verror": {
                   "version": "1.1.0",
-                  "from": "verror@1.1.0",
+                  "from": "https://registry.npmjs.org/verror/-/verror-1.1.0.tgz",
                   "resolved": "https://registry.npmjs.org/verror/-/verror-1.1.0.tgz",
                   "dependencies": {
                     "extsprintf": {
                       "version": "1.0.0",
-                      "from": "extsprintf@1.0.0",
+                      "from": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.0.tgz",
                       "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.0.tgz"
                     }
                   }
@@ -1780,11 +1795,6 @@
               }
             }
           }
-        },
-        "dtrace-provider": {
-          "version": "0.2.8",
-          "from": "dtrace-provider@0.2.8",
-          "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.2.8.tgz"
         }
       }
     },
@@ -1950,23 +1960,6 @@
       "from": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.3.tgz",
       "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.3.tgz"
     },
-    "nodejs-disks": {
-      "version": "0.2.1",
-      "from": "https://registry.npmjs.org/nodejs-disks/-/nodejs-disks-0.2.1.tgz",
-      "resolved": "https://registry.npmjs.org/nodejs-disks/-/nodejs-disks-0.2.1.tgz",
-      "dependencies": {
-        "async": {
-          "version": "0.2.10",
-          "from": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
-          "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz"
-        },
-        "numeral": {
-          "version": "1.4.8",
-          "from": "https://registry.npmjs.org/numeral/-/numeral-1.4.8.tgz",
-          "resolved": "https://registry.npmjs.org/numeral/-/numeral-1.4.8.tgz"
-        }
-      }
-    },
     "nodemailer": {
       "version": "1.3.4",
       "from": "https://registry.npmjs.org/nodemailer/-/nodemailer-1.3.4.tgz",
@@ -2273,9 +2266,9 @@
       "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.13.0.tgz"
     },
     "safetydance": {
-      "version": "0.0.16",
-      "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.16.tgz",
-      "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.16.tgz"
+      "version": "0.0.19",
+      "from": "safetydance@0.0.19",
+      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
     },
     "semver": {
       "version": "4.3.6",
@@ -2448,11 +2441,6 @@
         }
       }
     },
-    "supervisord-eventlistener": {
-      "version": "0.1.0",
-      "from": "https://registry.npmjs.org/supervisord-eventlistener/-/supervisord-eventlistener-0.1.0.tgz",
-      "resolved": "https://registry.npmjs.org/supervisord-eventlistener/-/supervisord-eventlistener-0.1.0.tgz"
-    },
     "tail-stream": {
       "version": "0.2.1",
       "from": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",

package.json

@@ -17,10 +17,11 @@
   },
   "dependencies": {
     "async": "^1.2.1",
+    "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^1.4.0",
+    "cloudron-manifestformat": "^1.7.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.12",
+    "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
     "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
@@ -43,7 +44,6 @@
     "mysql": "^2.7.0",
     "native-dns": "^0.7.0",
     "node-uuid": "^1.4.3",
-    "nodejs-disks": "^0.2.1",
     "nodemailer": "^1.3.0",
     "nodemailer-smtp-transport": "^1.0.3",
     "oauth2orize": "^1.0.1",
@@ -55,13 +55,12 @@
     "passport-oauth2-client-password": "^0.1.2",
     "password-generator": "^1.0.0",
     "proxy-middleware": "^0.13.0",
-    "safetydance": "0.0.16",
+    "safetydance": "0.0.19",
     "semver": "^4.3.6",
     "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
     "superagent": "~0.21.0",
     "supererror": "^0.7.0",
-    "supervisord-eventlistener": "^0.1.0",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "underscore": "^1.7.0",
     "valid-url": "^1.0.9",
@@ -69,7 +68,6 @@
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "^2.1.10",
     "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",
@@ -92,9 +90,9 @@
     "yargs": "^3.15.0"
   },
   "scripts": {
-    "migrate_local": "NODE_ENV=local DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
-    "migrate_test": "NODE_ENV=test DATABASE_URL=mysql://root:@localhost/boxtest node_modules/.bin/db-migrate up",
-    "test": "npm run migrate_test && src/test/setupTest && NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
+    "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
+    "migrate_test": "BOX_ENV=test DATABASE_URL=mysql://root:@localhost/boxtest node_modules/.bin/db-migrate up",
+    "test": "npm run migrate_test && src/test/setupTest && BOX_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
     "postmerge": "/bin/true",
     "precommit": "/bin/true",
     "prepush": "npm test",

setup/DESIGN.md

@@ -16,7 +16,7 @@ and replace it with a new one for an update.
 
 Because we do not package things as Docker yet, we should be careful
 about the code here. We have to expect remains of an older setup code.
-For example, older supervisor or nginx configs might be around.
+For example, older systemd or nginx configs might be around.
 
 The config directory is _part_ of the container and is not a VOLUME.
 Which is to say that the files will be nuked from one update to the next.
@@ -40,7 +40,7 @@ version (see below) or the mysql/postgresql data etc.
 
   * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
 
-  * supervisor is then started
+  * box services are then started
 
 setup_infra.sh
 This setups containers like graphite, mail and the addons containers.

setup/INFRA_VERSION

@@ -3,4 +3,15 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=4
+INFRA_VERSION=8
+
+# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+# These constants are used in the installer script as well
+BASE_IMAGE=cloudron/base:0.3.3
+MYSQL_IMAGE=cloudron/mysql:0.3.3
+POSTGRESQL_IMAGE=cloudron/postgresql:0.3.2
+MONGODB_IMAGE=cloudron/mongodb:0.3.2
+REDIS_IMAGE=cloudron/redis:0.3.2 # if you change this, fix src/addons.js as well
+MAIL_IMAGE=cloudron/mail:0.3.2
+GRAPHITE_IMAGE=cloudron/graphite:0.3.4
+

setup/argparser.sh

@@ -3,19 +3,21 @@
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 json="${script_dir}/../node_modules/.bin/json"
 
-arg_restore_url=""
-arg_restore_key=""
+# IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
+arg_api_server_origin=""
 arg_box_versions_url=""
+arg_fqdn=""
+arg_is_custom_domain="false"
+arg_restore_key=""
+arg_restore_url=""
+arg_retire="false"
 arg_tls_cert=""
 arg_tls_key=""
-arg_api_server_origin=""
-arg_web_server_origin=""
-arg_fqdn=""
 arg_token=""
 arg_version=""
-arg_is_custom_domain="false"
-arg_retire="false"
-arg_model=""
+arg_web_server_origin=""
+arg_backup_key=""
+arg_aws=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -41,8 +43,11 @@ EOF
         arg_restore_key=$(echo "$2" | $json restoreKey)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
-        arg_model=$(echo "$2" | $json model)
-        [[ "${arg_model}" == "null" ]] && arg_model=""
+        arg_backup_key=$(echo "$2" | $json backupKey)
+        [[ "${arg_backup_key}" == "null" ]] && arg_backup_key=""
+
+        arg_aws=$(echo "$2" | $json aws)
+        [[ "${arg_aws}" == "null" ]] && arg_aws=""
 
         shift 2
         ;;
@@ -52,15 +57,14 @@ EOF
 done
 
 echo "Parsed arguments:"
-echo "restore url: ${arg_restore_url}"
-echo "restore key: ${arg_restore_key}"
-echo "box versions url: ${arg_box_versions_url}"
 echo "api server: ${arg_api_server_origin}"
-echo "web server: ${arg_web_server_origin}"
+echo "box versions url: ${arg_box_versions_url}"
 echo "fqdn: ${arg_fqdn}"
-echo "token: ${arg_token}"
-echo "version: ${arg_version}"
 echo "custom domain: ${arg_is_custom_domain}"
+echo "restore key: ${arg_restore_key}"
+echo "restore url: ${arg_restore_url}"
 echo "tls cert: ${arg_tls_cert}"
 echo "tls key: ${arg_tls_key}"
-echo "model: ${arg_model}"
+echo "token: ${arg_token}"
+echo "version: ${arg_version}"
+echo "web server: ${arg_web_server_origin}"

setup/container.sh

@@ -13,13 +13,10 @@ readonly DATA_DIR="/home/yellowtent/data"
 rm -rf "${CONFIG_DIR}"
 sudo -u yellowtent mkdir "${CONFIG_DIR}"
 
-########## logrotate (default ubuntu runs this daily)
-rm -rf /etc/logrotate.d/*
-cp -r "${container_files}/logrotate/." /etc/logrotate.d/
-
-########## supervisor
-rm -rf /etc/supervisor/*
-cp -r "${container_files}/supervisor/." /etc/supervisor/
+########## systemd
+cp -r "${container_files}/systemd/." /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable cloudron.target
 
 ########## sudoers
 rm /etc/sudoers.d/*
@@ -34,6 +31,9 @@ ln -sfF "${DATA_DIR}/collectd" /etc/collectd
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
 ln -s "${DATA_DIR}/nginx" /etc/nginx
 
+########## mysql
+cp "${container_files}/mysql.cnf" /etc/mysql/mysql.cnf
+
 ########## Enable services
 update-rc.d -f collectd defaults
 

setup/container/logrotate/cloudron

@@ -1,6 +0,0 @@
-/var/log/cloudron/*log {
-    missingok
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/logrotate/supervisor

@@ -1,7 +0,0 @@
-/var/log/supervisor/*log {
-    missingok
-    copytruncate
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/mysql.cnf

@@ -0,0 +1,7 @@
+!includedir /etc/mysql/conf.d/
+!includedir /etc/mysql/mysql.conf.d/
+
+# http://bugs.mysql.com/bug.php?id=68514
+[mysqld]
+performance_schema=OFF
+max_connection=50

setup/container/sudoers

@@ -1,26 +1,29 @@
-Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
 
-Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmappdir.sh
 
-Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
 
-Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupbox.sh
 
-Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupapp.sh
 
-Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restoreapp.sh
 
-Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
 
-Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
 
-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME NODE_ENV"
+Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
+
+Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

setup/container/supervisor/conf.d/apphealthtask.conf

@@ -1,10 +0,0 @@
-[program:apphealthtask]
-command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"

setup/container/supervisor/conf.d/box.conf

@@ -1,10 +0,0 @@
-[program:box]
-command=/usr/bin/node "/home/yellowtent/box/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",NODE_ENV="cloudron"

setup/container/supervisor/conf.d/crashnotifier.conf

@@ -1,11 +0,0 @@
-[eventlistener:crashnotifier]
-command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
-events=PROCESS_STATE
-autostart=true
-autorestart=true
-redirect_stderr=false
-stderr_logfile=/var/log/supervisor/crashnotifier.log
-stderr_logfile_maxbytes=50MB
-stderr_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",NODE_ENV="cloudron"

setup/container/supervisor/conf.d/janitor.conf

@@ -1,10 +0,0 @@
-[program:janitor]
-command=/usr/bin/node "/home/yellowtent/box/janitor.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/janitor.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"

setup/container/supervisor/conf.d/oauthproxy.conf

@@ -1,10 +0,0 @@
-[program:oauthproxy]
-command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/oauthproxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"

setup/container/supervisor/supervisord.conf

@@ -1,33 +0,0 @@
-; supervisor config file
-
-; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[supervisord]
-logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
-pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-nodaemon = false
-childlogdir = /var/log/supervisor/
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=http://127.0.0.1:9001
-
-; The [include] section can just contain the "files" setting.  This
-; setting can list multiple files (separated by whitespace or
-; newlines).  It can also contain wildcards.  The filenames are
-; interpreted as relative to this file.  Included files *cannot*
-; include files themselves.
-
-[include]
-files = conf.d/*.conf
-

setup/container/systemd/apphealthtask.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Cloudron App Health Monitor
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/apphealthtask.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/container/systemd/box.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/app.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smart Cloud
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=apphealthtask.service box.service janitor.service oauthproxy.service
+After=apphealthtask.service box.service janitor.service oauthproxy.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+

setup/container/systemd/janitor.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Cloudron Janitor
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/janitor.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M
+

setup/container/systemd/oauthproxy.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Cloudron OAuth Proxy Service
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart="/home/yellowtent/box/oauthproxy.js"
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M
+

setup/splashpage.sh

@@ -7,6 +7,7 @@ readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
 source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
 
@@ -14,6 +15,10 @@ echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
 # copy the website
 rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
@@ -24,14 +29,10 @@ infra_version="none"
 if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
-    # keep this is sync with config.js appFqdn()
-    readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
-    admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
-
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi
 
 echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"

setup/start.sh

@@ -19,6 +19,7 @@ source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used
 
 # keep this is sync with config.js appFqdn()
 admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
 
 readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
 
@@ -40,6 +41,9 @@ mkdir -p "${DATA_DIR}/box/appicons"
 mkdir -p "${DATA_DIR}/box/mail"
 mkdir -p "${DATA_DIR}/graphite"
 
+mkdir -p "${DATA_DIR}/mysql"
+mkdir -p "${DATA_DIR}/postgresql"
+mkdir -p "${DATA_DIR}/mongodb"
 mkdir -p "${DATA_DIR}/snapshots"
 mkdir -p "${DATA_DIR}/addons"
 mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
@@ -52,6 +56,9 @@ echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_version
 echo "Cleaning up snapshots"
 find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
 
+# restart mysql to make sure it has latest config
+service mysql restart
+
 readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
 mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
@@ -77,11 +84,15 @@ set_progress "25" "Migrating data"
 sudo -u "${USER}" -H bash <<EOF
 set -eu
 cd "${BOX_SRC_DIR}"
-NODE_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
+BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
 EOF
 
 set_progress "28" "Setup collectd"
 cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
+# collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
+mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
+vda1_id=$(blkid -s UUID -o value /dev/vda1)
+ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
 service collectd restart
 
 set_progress "30" "Setup nginx"
@@ -95,7 +106,7 @@ ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs
 
 # generate these for update code paths as well to overwrite splash
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
 mkdir -p "${DATA_DIR}/nginx/cert"
 echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
@@ -108,10 +119,10 @@ set_progress "40" "Setting up infra"
 ${script_dir}/start/setup_infra.sh "${arg_fqdn}"
 
 set_progress "65" "Creating cloudron.conf"
-admin_origin="https://${admin_fqdn}"
 sudo -u yellowtent -H bash <<EOF
 set -eu
 echo "Creating cloudron.conf"
+# note that arg_aws is a javascript object and intentionally unquoted below
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
@@ -129,7 +140,8 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "port": 3306,
         "name": "box"
     },
-    "model": "${arg_model}"
+    "backupKey": "${arg_backup_key}",
+    "aws": ${arg_aws}
 }
 CONF_END
 
@@ -154,22 +166,10 @@ ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
 mysql -u root -p${mysql_root_password} \
     -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "80" "Reloading supervisor"
-# looks like restarting supervisor completely is the only way to reload it
-service supervisor stop || true
+set_progress "80" "Starting Cloudron"
+systemctl start cloudron.target
 
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
+sleep 2 # give systemd sometime to start the processes
 
 set_progress "85" "Reloading nginx"
 nginx -s reload

setup/start/collectd.conf

@@ -193,12 +193,11 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin df>
-   Device "/dev/vda1"
-   Device "/dev/loop0"
-   Device "/dev/loop1"
+   FSType "tmpfs"
+   MountPoint "/dev"
 
    ReportByDevice true
-   IgnoreSelected false
+   IgnoreSelected true
 
    ValuesAbsolute true
    ValuesPercentage true

setup/start/nginx/appconfig.ejs

@@ -37,11 +37,9 @@ server {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
 
-    error_page 500 502 503 504 =200 @appstatus;
+    error_page 500 502 503 504 @appstatus;
     location @appstatus {
-        internal;
-        root <%= sourceDir %>/webadmin/dist;
-        rewrite ^/$ /appstatus.html break;
+        return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
     }
 
     location / {

setup/start/setup_infra.sh

@@ -27,11 +27,13 @@ if [[ -n "${existing_containers}" ]]; then
 fi
 
 # graphite
-docker run --restart=always -d --name="graphite" \
+graphite_container_id=$(docker run --restart=always -d --name="graphite" \
     -p 127.0.0.1:2003:2003 \
     -p 127.0.0.1:2004:2004 \
     -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" cloudron/graphite:0.3.1
+    -v "${DATA_DIR}/graphite:/app/data" \
+    "${GRAPHITE_IMAGE}")
+echo "Graphite container id: ${graphite_container_id}"
 
 # mail
 mail_container_id=$(docker run --restart=always -d --name="mail" \
@@ -39,7 +41,7 @@ mail_container_id=$(docker run --restart=always -d --name="mail" \
     -h "${arg_fqdn}" \
     -e "DOMAIN_NAME=${arg_fqdn}" \
     -v "${DATA_DIR}/box/mail:/app/data" \
-    cloudron/mail:0.3.0)
+    "${MAIL_IMAGE}")
 echo "Mail container id: ${mail_container_id}"
 
 # mysql
@@ -52,8 +54,8 @@ EOF
 mysql_container_id=$(docker run --restart=always -d --name="mysql" \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:r" \
-    cloudron/mysql:0.3.0)
+    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+    "${MYSQL_IMAGE}")
 echo "MySQL container id: ${mysql_container_id}"
 
 # postgresql
@@ -64,8 +66,8 @@ EOF
 postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:r" \
-    cloudron/postgresql:0.3.0)
+    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+    "${POSTGRESQL_IMAGE}")
 echo "PostgreSQL container id: ${postgresql_container_id}"
 
 # mongodb
@@ -76,8 +78,8 @@ EOF
 mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:r" \
-    cloudron/mongodb:0.3.0)
+    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+    "${MONGODB_IMAGE}")
 echo "Mongodb container id: ${mongodb_container_id}"
 
 if [[ "${infra_version}" == "none" ]]; then

setup/stop.sh

@@ -2,14 +2,6 @@
 
 set -eu -o pipefail
 
-echo "Stopping box code"
-
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
+echo "Stopping cloudron"
 
+systemctl stop cloudron.target

src/addons.js

@@ -38,8 +38,7 @@ var appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
     util = require('util'),
     uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
-    _ = require('underscore');
+    vbox = require('./vbox.js');
 
 var NOOP = function (app, callback) { return callback(); };
 
@@ -665,7 +664,7 @@ function setupRedis(app, callback) {
         name: 'redis-' + app.id,
         Hostname: config.appFqdn(app.location),
         Tty: true,
-        Image: 'cloudron/redis:0.3.0',
+        Image: 'cloudron/redis:0.3.2', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
         Volumes: {},
         VolumesFrom: []
@@ -675,7 +674,7 @@ function setupRedis(app, callback) {
 
     var startOptions = {
         Binds: [
-            redisVarsFile + ':/etc/redis/redis_vars.sh:r',
+            redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
             redisDataDir + ':/var/lib/redis:rw'
         ],
         // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work

src/appdb.js

@@ -35,12 +35,13 @@ exports = module.exports = {
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
-    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
     RSTATE_STOPPED: 'stopped', // app stopped by use
+    RSTATE_ERROR: 'error',
 
+    // run codes (keep in sync in UI)
     HEALTH_HEALTHY: 'healthy',
     HEALTH_UNHEALTHY: 'unhealthy',
     HEALTH_ERROR: 'error',
@@ -335,6 +336,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
 
     // Rules are:
     // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
     // update and configure are allowed only in installed state
 

src/apps.js

@@ -490,28 +490,30 @@ function restore(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var restoreConfig = app.lastBackupConfig;
-        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+        // restore without a backup is the same as re-install
+        var restoreConfig = app.lastBackupConfig, values = { };
+        if (restoreConfig) {
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
-        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
-        error = checkManifestConstraints(restoreConfig.manifest);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
+            if (error) return callback(error);
 
-        error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
-        if (error) return callback(error);
+            // ## should probably query new location, access restriction from user
+            values = {
+                manifest: restoreConfig.manifest,
+                portBindings: restoreConfig.portBindings,
 
-        // ## should probably query new location, access restriction from user
-        var values = {
-            manifest: restoreConfig.manifest,
-            portBindings: restoreConfig.portBindings,
-
-            oldConfig: {
-                location: app.location,
-                accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings,
-                manifest: app.manifest
-            }
-        };
+                oldConfig: {
+                    location: app.location,
+                    accessRestriction: app.accessRestriction,
+                    portBindings: app.portBindings,
+                    manifest: app.manifest
+                }
+            };
+        }
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -573,6 +575,8 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
         return new Error('Box version exceeds Apps maxBoxVersion');
     }
@@ -664,7 +668,7 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
                 return iteratorDone();
             }
 
-           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
                 if (error) debug('Error initiating autoupdate of %s', appId);
 
                 iteratorDone(null);
@@ -675,16 +679,16 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
 
 function backupApp(app, addonsToBackup, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof addonsToBackup, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function canBackupApp(app) {
         // only backup apps that are installed or pending configure. Rest of them are in some
         // state not good for consistent backup (i.e addons may not have been setup completely)
-        return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY)
-                || app.installationState === appdb.ISTATE_PENDING_CONFIGURE
-                || app.installationState === appdb.ISTATE_PENDING_BACKUP
-                || app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+        return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+                app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+                app.installationState === appdb.ISTATE_PENDING_BACKUP ||
+                app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
     }
 
     if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy'));
@@ -700,7 +704,7 @@ function backupApp(app, addonsToBackup, callback) {
         return callback(safe.error);
     }
 
-    backups.getBackupUrl(app, null, function (error, result) {
+    backups.getBackupUrl(app, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -709,7 +713,7 @@ function backupApp(app, addonsToBackup, callback) {
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
             addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -756,7 +760,7 @@ function restoreApp(app, addonsToRestore, callback) {
 
         debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
 
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             addons.restoreAddons(app, addonsToRestore, callback);

src/apptask.js

@@ -46,6 +46,7 @@ var addons = require('./addons.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
@@ -93,7 +94,7 @@ function configureNginx(app, callback) {
 
         var sourceDir = path.resolve(__dirname, '..');
         var endpoint = app.accessRestriction ? 'oauthproxy' : 'app';
-        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, { sourceDir: sourceDir, vhost: config.appFqdn(app.location), port: freePort, endpoint: endpoint });
+        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, { sourceDir: sourceDir, adminOrigin: config.adminOrigin(), vhost: config.appFqdn(app.location), port: freePort, endpoint: endpoint });
 
         var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
         debugApp(app, 'writing config to %s', nginxConfigFilename);
@@ -189,7 +190,6 @@ function createContainer(app, callback) {
         }
 
         env.push('CLOUDRON=1');
-        env.push('ADMIN_ORIGIN' + '=' + config.adminOrigin()); // ## remove
         env.push('WEBADMIN_ORIGIN' + '=' + config.adminOrigin());
         env.push('API_ORIGIN' + '=' + config.adminOrigin());
 
@@ -202,8 +202,6 @@ function createContainer(app, callback) {
                 Tty: true,
                 Image: app.manifest.dockerImage,
                 Cmd: null,
-                Volumes: {},
-                VolumesFrom: [],
                 Env: env.concat(addonEnv),
                 ExposedPorts: exposedPorts
             };
@@ -251,7 +249,7 @@ function deleteImage(app, manifest, callback) {
             noprune: false
         };
 
-        // delete image by id because docker pull pulls down all the tags and this is the only way to delete all tags
+         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
         docker.getImage(result.Id).remove(removeOptions, function (error) {
             if (error && error.statusCode === 404) return callback(null);
             if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -334,15 +332,20 @@ function startContainer(app, callback) {
             vbox.forwardFromHostToVirtualBox(app.id + '-tcp' + containerPort, hostPort);
         }
 
+        var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+
         var startOptions = {
             Binds: addons.getBindsSync(app, app.manifest.addons),
+            Memory: memoryLimit / 2,
+            MemorySwap: memoryLimit, // Memory + Swap
             PortBindings: dockerPortBindings,
             PublishAllPorts: false,
             Links: addons.getLinksSync(app, app.manifest.addons),
             RestartPolicy: {
                 "Name": "always",
                 "MaximumRetryCount": 0
-            }
+            },
+            CpuShares: 512 // relative to 1024 for system processes
         };
 
         var container = docker.getContainer(app.containerId);
@@ -357,6 +360,11 @@ function startContainer(app, callback) {
 }
 
 function stopContainer(app, callback) {
+    if (!app.containerId) {
+        debugApp(app, 'No previous container to stop');
+        return callback();
+    }
+
     var container = docker.getContainer(app.containerId);
     debugApp(app, 'Stopping container %s', container.id);
 
@@ -422,43 +430,27 @@ function registerSubdomain(app, callback) {
     // need to register it so that we have a dnsRecordId to wait for it to complete
     var record = { subdomain: app.location, type: 'A', value: sysinfo.getIp() };
 
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: [ record ] })
-        .end(function (error, res) {
-            if (error) return callback(error);
+    subdomains.add(record, function (error, changeId) {
+        if (error) return callback(error);
 
-            debugApp(app, 'Registered subdomain status: %s', res.status);
+        debugApp(app, 'Registered subdomain.');
 
-            if (res.status === 409) return callback(null); // already registered
-            if (res.status !== 201) return callback(new Error(util.format('Subdomain Registration failed. %s %j', res.status, res.body)));
-
-            updateApp(app, { dnsRecordId: res.body.ids[0] }, callback);
-        });
+        updateApp(app, { dnsRecordId: changeId }, callback);
+    });
 }
 
-function unregisterSubdomain(app, callback) {
-    debugApp(app, 'Unregistering subdomain: dnsRecordId=%s', app.dnsRecordId);
-
-    if (!app.dnsRecordId) return callback(null);
+function unregisterSubdomain(app, location, callback) {
+    debugApp(app, 'Unregistering subdomain: %s', location);
 
     // do not unregister bare domain because we show a error/cloudron info page there
-    if (app.location === '') return updateApp(app, { dnsRecordId: null }, callback);
+    if (location === '') return callback(null);
 
-    superagent
-        .del(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId)
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) {
-                debugApp(app, 'Error making request: %s', error);
-            } else if (res.status !== 204) {
-                debugApp(app, 'Error unregistering subdomain:', res.status, res.body);
-            }
+    var record = { subdomain: location, type: 'A', value: sysinfo.getIp() };
+    subdomains.remove(record, function (error) {
+        if (error) debugApp(app, 'Error unregistering subdomain: %s', error);
 
-            updateApp(app, { dnsRecordId: null }, callback);
-        });
+        updateApp(app, { dnsRecordId: null }, callback);
+    });
 }
 
 function removeIcon(app, callback) {
@@ -479,21 +471,15 @@ function waitForDnsPropagation(app, callback) {
         setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
     }
 
-    superagent
-        .get(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId + '/status')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+    subdomains.status(app.dnsRecordId, function (error, result) {
+        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
 
-            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, res.status);
+        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-            if (res.status !== 200) return retry(new Error(util.format('Error getting record status: %s %j', res.status, res.body)));
+        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
 
-            if (res.body.status !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, res.body.status)));
-
-            callback(null);
-        });
+        callback(null);
+    });
 }
 
 // updates the app object and the database
@@ -532,9 +518,9 @@ function install(app, callback) {
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
         deleteVolume.bind(null, app),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
         removeOAuthProxyCredentials.bind(null, app),
-        removeIcon.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
         unconfigureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
@@ -619,7 +605,11 @@ function restore(app, callback) {
          // oldConfig can be null during upgrades
         addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
         deleteVolume.bind(null, app),
-        deleteImage.bind(null, app, app.manifest),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -673,16 +663,15 @@ function restore(app, callback) {
 
 // note that configure is called after an infra update as well
 function configure(app, callback) {
-    var locationChanged = app.oldConfig.location !== app.location;
-
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         function (next) {
-            if (!locationChanged) return next();
-            unregisterSubdomain(app, next);
+            // oldConfig can be null during an infra update
+            if (!app.oldConfig || app.oldConfig.location === app.location) return next();
+            unregisterSubdomain(app, app.oldConfig.location, next);
         },
         removeOAuthProxyCredentials.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -693,14 +682,8 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
         allocateOAuthProxyCredentials.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
-                registerSubdomain.bind(null, app)
-            ], next);
-        },
+        updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
+        registerSubdomain.bind(null, app),
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         updateApp.bind(null, app, { installationProgress: '50, Setting up addons' }),
@@ -714,14 +697,8 @@ function configure(app, callback) {
 
         runApp.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
-                updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
-                exports._waitForDnsPropagation.bind(null, app)
-            ], next);
-        },
+        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
+        exports._waitForDnsPropagation.bind(null, app),
 
         // done!
         function (callback) {
@@ -755,7 +732,11 @@ function update(app, callback) {
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, unusedAddons),
-        deleteImage.bind(null, app, app.manifest), // delete image even if did not change (see df158b111f)
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
 
         function (next) {
@@ -821,7 +802,7 @@ function uninstall(app, callback) {
         deleteImage.bind(null, app, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
 
         updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
         removeOAuthProxyCredentials.bind(null, app),

src/aws.js

@@ -0,0 +1,282 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    AWSError: AWSError,
+
+    getAWSCredentials: getAWSCredentials,
+
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl,
+
+    addSubdomain: addSubdomain,
+    delSubdomain: delSubdomain,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('./config.js'),
+    debug = require('debug')('box:aws'),
+    SubdomainError = require('./subdomainerror.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function AWSError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AWSError, Error);
+AWSError.INTERNAL_ERROR = 'Internal Error';
+AWSError.MISSING_CREDENTIALS = 'Missing AWS credentials';
+
+function getAWSCredentials(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
+        superagent.get(url).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 201) return callback(new Error(result.text));
+            if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
+
+            return callback(null, {
+                accessKeyId: result.body.credentials.AccessKeyId,
+                secretAccessKey: result.body.credentials.SecretAccessKey,
+                sessionToken: result.body.credentials.SessionToken,
+                region: 'us-east-1'
+            });
+        });
+    } else {
+        if (!config.aws().accessKeyId || !config.aws().secretAccessKey) return callback(new AWSError(AWSError.MISSING_CREDENTIALS));
+
+        callback(null, {
+            accessKeyId: config.aws().accessKeyId,
+            secretAccessKey: config.aws().secretAccessKey,
+            region: 'us-east-1'
+        });
+    }
+}
+
+function getSignedUploadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedUploadUrl()');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedDownloadUrl()');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getZoneByName(zoneName, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getZoneByName: %s', zoneName);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.listHostedZones({}, function (error, result) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+
+            var zone = result.HostedZones.filter(function (zone) {
+                return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+            })[0];
+
+            if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
+
+            debug('getZoneByName: found zone', zone);
+
+            callback(null, zone);
+        });
+    });
+}
+
+function addSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addSubdomain: ' + subdomain + ' for domain ' + zoneName + ' with value ' + value);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'UPSERT',
+                    ResourceRecordSet: {
+                        Type: type,
+                        Name: fqdn,
+                        ResourceRecords: [{
+                            Value: value
+                        }],
+                        Weight: 0,
+                        SetIdentifier: fqdn,
+                        TTL: 1
+                    }
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.code === 'PriorRequestNotComplete') {
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                } else if (error) {
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                }
+
+                debug('addSubdomain: success. changeInfoId:%j', result);
+
+                callback(null, result.ChangeInfo.Id);
+            });
+        });
+    });
+}
+
+function delSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('delSubdomain: %s for domain %s.', subdomain, zoneName);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var resourceRecordSet = {
+            Name: fqdn,
+            Type: type,
+            ResourceRecords: [{
+                Value: value
+            }],
+            Weight: 0,
+            SetIdentifier: fqdn,
+            TTL: 1
+        };
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'DELETE',
+                    ResourceRecordSet: resourceRecordSet
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                    debug('delSubdomain: resource record set not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'NoSuchHostedZone') {
+                    debug('delSubdomain: hosted zone not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'PriorRequestNotComplete') {
+                    debug('delSubdomain: resource is still busy', error);
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                } else if (error && error.code === 'InvalidChangeBatch') {
+                    debug('delSubdomain: invalid change batch. No such record to be deleted.');
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error) {
+                    debug('delSubdomain: error', error);
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                }
+
+                debug('delSubdomain: success');
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function getChangeStatus(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.getChange({ Id: changeId }, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, result.ChangeInfo.Status);
+        });
+    });
+}

src/backups.js

@@ -10,6 +10,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    aws = require('./aws.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
     superagent = require('superagent'),
@@ -54,42 +55,50 @@ function getAllPaged(page, perPage, callback) {
     });
 }
 
-function getBackupUrl(app, appBackupIds, callback) {
+function getBackupUrl(app, callback) {
     assert(!app || typeof app === 'object');
-    assert(!appBackupIds || util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+    var filename = '';
+    if (app) {
+        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    } else {
+        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
+    }
 
-    var data = {
-        boxVersion: config.version(),
-        appId: app ? app.id : null,
-        appVersion: app ? app.manifest.version : null,
-        appBackupIds: appBackupIds
-    };
+    aws.getSignedUploadUrl(filename, function (error, result) {
+        if (error) return callback(error);
 
-    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        var obj = {
+            id: filename,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getBackupUrl: ', obj);
+
+        callback(null, obj);
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+    aws.getSignedDownloadUrl(backupId, function (error, result) {
+        if (error) return callback(error);
 
-    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        var obj = {
+            id: backupId,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getRestoreUrl: ', obj);
+
+        callback(null, obj);
     });
 }
-
-

src/cloudron.js

@@ -19,8 +19,7 @@ exports = module.exports = {
     reboot: reboot,
     migrate: migrate,
     backup: backup,
-    ensureBackup: ensureBackup
-};
+    ensureBackup: ensureBackup};
 
 var apps = require('./apps.js'),
     AppsError = require('./apps.js').AppsError,
@@ -40,6 +39,7 @@ var apps = require('./apps.js'),
     settings = require('./settings.js'),
     SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
@@ -47,7 +47,8 @@ var apps = require('./apps.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
 
 var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
@@ -55,7 +56,7 @@ var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
 
-var gAddMailDnsRecordsTimerId = null,
+var gAddDnsRecordsTimerId = null,
     gCloudronDetails = null;            // cached cloudron details like region,size...
 
 function debugApp(app, args) {
@@ -108,21 +109,18 @@ CloudronError.NOT_FOUND = 'Not found';
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (process.env.NODE_ENV !== 'test') {
-        addMailDnsRecords();
+    if (process.env.BOX_ENV !== 'test') {
+        addDnsRecords();
     }
 
-    // Send heartbeat once we are up and running, this speeds up the Cloudron creation, as otherwise we are bound to the cron.js settings
-    sendHeartbeat();
-
     callback(null);
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clearTimeout(gAddMailDnsRecordsTimerId);
-    gAddMailDnsRecordsTimerId = null;
+    clearTimeout(gAddDnsRecordsTimerId);
+    gAddDnsRecordsTimerId = null;
 
     callback(null);
 }
@@ -270,19 +268,20 @@ function getConfig(callback) {
 }
 
 function sendHeartbeat() {
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
-    debug('Sending heartbeat ' + url);
+    // Only send heartbeats after the admin dns record is synced to give appstore a chance to know that fact
+    if (!config.get('dnsInSync')) return;
 
-    // TODO: this must be a POST
-    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
+
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
         if (error) debug('Error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
-        else debug('Heartbeat successful');
+        else debug('Heartbeat sent to %s', url);
     });
 }
 
-function sendMailDnsRecordsRequest(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function addDnsRecords() {
+    if (config.get('dnsInSync')) return sendHeartbeat(); // already registered send heartbeat
 
     var DKIM_SELECTOR = 'mail';
     var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
@@ -290,13 +289,20 @@ function sendMailDnsRecordsRequest(callback) {
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
 
-    if (publicKey === null) return callback(new Error('Error reading dkim public key'));
+    if (publicKey === null) {
+        console.error('Error reading dkim public key. Stop DNS setup.');
+        return;
+    }
 
     // remove header, footer and new lines
     publicKey = publicKey.split('\n').slice(1, -2).join('');
 
     // note that dmarc requires special DNS records for external RUF and RUA
     var records = [
+        // naked domain
+        { subdomain: '', type: 'A', value: sysinfo.getIp() },
+        // webadmin domain
+        { subdomain: 'my', type: 'A', value: sysinfo.getIp() },
         // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
         { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
         // t=s limits the domainkey to this domain and not it's subdomains
@@ -305,38 +311,47 @@ function sendMailDnsRecordsRequest(callback) {
         { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
     ];
 
-    debug('sendMailDnsRecords request:%s', JSON.stringify(records));
+    debug('addDnsRecords:', records);
 
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: records })
-        .end(function (error, res) {
-            if (error) return callback(error);
-
-            debug('sendMailDnsRecords status: %s', res.status);
-
-            if (res.status === 409) return callback(null); // already registered
-
-            if (res.status !== 201) return callback(new Error(util.format('Failed to add Mail DNS records: %s %j', res.status, res.body)));
-
-            return callback(null, res.body.ids);
-        });
-}
-
-function addMailDnsRecords() {
-    if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
-
-    sendMailDnsRecordsRequest(function (error, ids) {
+    subdomains.addMany(records, function (error, changeIds) {
         if (error) {
-            console.error('Mail DNS record addition failed', error);
-            gAddMailDnsRecordsTimerId = setTimeout(addMailDnsRecords, 30000);
+            console.error('Admin DNS record addition failed', error);
+            gAddDnsRecordsTimerId = setTimeout(addDnsRecords, 10000);
             return;
         }
 
-        debug('Added Mail DNS records successfully');
-        config.set('mailDnsRecordIds', ids);
+        function checkIfInSync() {
+            debug('addDnsRecords: Check if admin DNS record is in sync.');
+
+            var allDone = true;
+
+            async.each(changeIds, function (changeId, callback) {
+                subdomains.status(changeId, function (error, result) {
+                    if (error) return callback(new Error('Failed to check if admin DNS record is in sync.', error));
+
+                    if (result !== 'done') allDone = false;
+
+                    callback(null);
+                });
+            }, function (error) {
+                if (error) console.error(error);
+
+                // retry if needed
+                if (error || !allDone) {
+                    gAddDnsRecordsTimerId = setTimeout(checkIfInSync, 5000);
+                    return;
+                }
+
+                config.set('dnsInSync', true);
+
+                // send heartbeat after the dns records are done
+                sendHeartbeat();
+
+                debug('addDnsRecords: done');
+            });
+        }
+
+        checkIfInSync();
     });
 }
 
@@ -418,16 +433,22 @@ function update(boxUpdateInfo, callback) {
     var error = locker.lock(locker.OP_BOX_UPDATE);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
-    progress.set(progress.UPDATE, 0, 'Begin ' + (boxUpdateInfo.update ? 'upgrade': 'update'));
-
     // initiate the update/upgrade but do not wait for it
     if (boxUpdateInfo.upgrade) {
+        debug('Starting upgrade');
         doUpgrade(boxUpdateInfo, function (error) {
-            locker.unlock(locker.OP_BOX_UPDATE);
+            if (error) {
+                debug('Upgrade failed with error: %s', error);
+                locker.unlock(locker.OP_BOX_UPDATE);
+            }
         });
     } else {
+        debug('Starting update');
         doUpdate(boxUpdateInfo, function (error) {
-            locker.unlock(locker.OP_BOX_UPDATE);
+            if (error) {
+                debug('Update failed with error: %s', error);
+                locker.unlock(locker.OP_BOX_UPDATE);
+            }
         });
     }
 
@@ -437,17 +458,22 @@ function update(boxUpdateInfo, callback) {
 function doUpgrade(boxUpdateInfo, callback) {
     assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
 
-    progress.set(progress.UPDATE, 5, 'Create app and box backup');
+    function upgradeError(e) {
+        progress.set(progress.UPDATE, -1, e.message);
+        callback(e);
+    }
+
+    progress.set(progress.UPDATE, 5, 'Create app and box backup for upgrade');
 
     backupBoxAndApps(function (error) {
-        if (error) return callback(error);
+        if (error) return upgradeError(error);
 
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
           .query({ token: config.token() })
           .send({ version: boxUpdateInfo.version })
           .end(function (error, result) {
-            if (error) return callback(new Error('Error making upgrade request: ' + error));
-            if (result.status !== 202) return callback(new Error('Server not ready to upgrade: ' + result.body));
+            if (error) return upgradeError(new Error('Error making upgrade request: ' + error));
+            if (result.status !== 202) return upgradeError(new Error('Server not ready to upgrade: ' + result.body));
 
             progress.set(progress.UPDATE, 10, 'Updating base system');
 
@@ -461,18 +487,23 @@ function doUpgrade(boxUpdateInfo, callback) {
 function doUpdate(boxUpdateInfo, callback) {
     assert(boxUpdateInfo && typeof boxUpdateInfo === 'object');
 
-    progress.set(progress.UPDATE, 5, 'Create box backup');
+    function updateError(e) {
+        progress.set(progress.UPDATE, -1, e.message);
+        callback(e);
+    }
+
+    progress.set(progress.UPDATE, 5, 'Create box backup for update');
 
     backupBox(function (error) {
-        if (error) return callback(error);
+        if (error) return updateError(error);
 
         // fetch a signed sourceTarballUrl
         superagent.get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/sourcetarballurl')
           .query({ token: config.token(), boxVersion: boxUpdateInfo.version })
           .end(function (error, result) {
-            if (error) return callback(new Error('Error fetching sourceTarballUrl: ' + error));
-            if (result.status !== 200) return callback(new Error('Error fetching sourceTarballUrl status: ' + result.status));
-            if (!safe.query(result, 'body.url')) return callback(new Error('Error fetching sourceTarballUrl response: ' + result.body));
+            if (error) return updateError(new Error('Error fetching sourceTarballUrl: ' + error));
+            if (result.status !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.status));
+            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + result.body));
 
             // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
             var args = {
@@ -490,15 +521,16 @@ function doUpdate(boxUpdateInfo, callback) {
                     tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
                     isCustomDomain: config.isCustomDomain(),
                     restoreUrl: null,
-                    restoreKey: null
+                    restoreKey: null,
+                    aws: config.aws()
                 }
             };
 
             debug('updating box %j', args);
 
             superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
-                if (error) return callback(error);
-                if (result.status !== 202) return callback(new Error('Error initiating update: ' + result.body));
+                if (error) return updateError(error);
+                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + result.body));
 
                 progress.set(progress.UPDATE, 10, 'Updating cloudron software');
 
@@ -547,22 +579,25 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+    backups.getBackupUrl(null /* app */, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new CloudronError.INTERNAL_ERROR, error);
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         debug('backup: url %s', result.url);
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             debug('backup: successful');
 
-            callback(null, result.id);
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
+                callback(null, result.id);
+            });
         });
     });
 }
@@ -573,7 +608,7 @@ function backupBox(callback) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null }); // remove apps that were never backed up
+        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
 
         backupBoxWithAppBackupIds(appBackupIds, callback);
     });
@@ -595,25 +630,27 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, app.location);
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
 
                 if (error && error.reason === AppsError.BAD_STATE) {
-                    debugApp(app, 'Skipping backup (istate:%s health%s). Reusing %s', app.installationState, app.health, app.lastBackupId);
+                    debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);
                     backupId = app.lastBackupId;
                 }
 
                 return iteratorCallback(null, backupId);
             });
         }, function appsBackedUp(error, backupIds) {
-            if (error) return callback(error);
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
 
             backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
 
             backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
-                progress.set(progress.BACKUP, 100, '');
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
                 callback(error, restoreKey);
             });
         });
     });
 }
-

src/collectd.config.ejs

@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
+  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.stat">
     Instance "<%= appId %>-memory"
     Separator " \\n"
     <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.max_usage_in_bytes">
     Instance "<%= appId %>-memory"
     Separator "\\n"
     <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker-<%= containerId %>.scope/cpuacct.stat">
     Instance "<%= appId %>-cpu"
     Separator " \\n"
     <Result>

src/config.js

@@ -11,8 +11,8 @@ exports = module.exports = {
     set: set,
 
     // ifdefs to check environment
-    CLOUDRON: process.env.NODE_ENV === 'cloudron',
-    TEST: process.env.NODE_ENV === 'test',
+    CLOUDRON: process.env.BOX_ENV === 'cloudron',
+    TEST: process.env.BOX_ENV === 'test',
 
     // convenience getters
     apiServerOrigin: apiServerOrigin,
@@ -30,6 +30,9 @@ exports = module.exports = {
 
     isDev: isDev,
 
+    backupKey: backupKey,
+    aws: aws,
+
     // for testing resets to defaults
     _reset: initConfig
 };
@@ -70,6 +73,14 @@ function initConfig() {
     data.webServerOrigin = null;
     data.internalPort = 3001;
     data.ldapPort = 3002;
+    data.backupKey = 'backupKey';
+    data.aws = {
+        backupBucket: null,
+        backupPrefix: null,
+        accessKeyId: null,      // selfhosting only
+        secretAccessKey: null   // selfhosting only
+    };
+    data.dnsInSync = false;
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -86,6 +97,7 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
+        data.aws.backupBucket = 'testbucket';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -99,6 +111,9 @@ function initConfig() {
     saveSync();
 }
 
+// cleanup any old config file we have for tests
+if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
+
 initConfig();
 
 // set(obj) or set(key, value)
@@ -172,3 +187,10 @@ function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
 
+function backupKey() {
+    return get('backupKey');
+}
+
+function aws() {
+    return get('aws');
+}

src/cron.js

@@ -98,12 +98,15 @@ function autoupdatePatternChanged(pattern) {
     gAutoupdaterJob = new CronJob({
         cronTime: pattern,
         onTick: function() {
-            debug('Starting autoupdate');
             var updateInfo = updateChecker.getUpdateInfo();
             if (updateInfo.box) {
+                debug('Starting autoupdate to %j', updateInfo.box);
                 cloudron.update(updateInfo.box, NOOP_CALLBACK);
             } else if (updateInfo.apps) {
+                debug('Starting app update to %j', updateInfo.apps);
                 apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+            } else {
+                debug('No auto updates available');
             }
         },
         start: true,

src/developer.js

@@ -7,12 +7,15 @@ exports = module.exports = {
 
     enabled: enabled,
     setEnabled: setEnabled,
-    issueDeveloperToken: issueDeveloperToken
+    issueDeveloperToken: issueDeveloperToken,
+    getNonApprovedApps: getNonApprovedApps
 };
 
 var assert = require('assert'),
+    config = require('./config.js'),
     tokendb = require('./tokendb.js'),
     settings = require('./settings.js'),
+    superagent = require('superagent'),
     util = require('util');
 
 function DeveloperError(reason, errorOrMessage) {
@@ -68,3 +71,15 @@ function issueDeveloperToken(user, callback) {
         callback(null, { token: token, expiresAt: expiresAt });
     });
 }
+
+function getNonApprovedApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
+    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        if (result.status !== 200) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+
+        callback(null, result.body.apps || []);
+    });
+}

src/docker.js

@@ -10,7 +10,7 @@ exports = module.exports = (function () {
     var docker;
     var options = connectOptions(); // the real docker
 
-    if (process.env.NODE_ENV === 'test') {
+    if (process.env.BOX_ENV === 'test') {
         // test code runs a docker proxy on this port
         docker = new Docker({ host: 'http://localhost', port: 5687 });
     } else {

src/ldap.js

@@ -24,6 +24,9 @@ var gLogger = {
     fatal: console.error
 };
 
+var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
+
 function start(callback) {
     assert(typeof callback === 'function');
 
@@ -39,15 +42,21 @@ function start(callback) {
             result.forEach(function (entry) {
                 var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
 
+                var groups = [ GROUP_USERS_DN ];
+                if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
                 var tmp = {
                     dn: dn.toString(),
                     attributes: {
                         objectclass: ['user'],
+                        objectcategory: 'person',
                         cn: entry.id,
                         uid: entry.id,
                         mail: entry.email,
                         displayname: entry.username,
-                        username: entry.username
+                        username: entry.username,
+                        samaccountname: entry.username,      // to support ActiveDirectory clients
+                        memberof: groups
                     }
                 };
 
@@ -67,22 +76,32 @@ function start(callback) {
         user.list(function (error, result){
             if (error) return next(new ldap.OperationsError(error.toString()));
 
-            // we only have an admin group
-            var dn = ldap.parseDN('cn=admin,ou=groups,dc=cloudron');
+            var groups = [{
+                name: 'users',
+                admin: false
+            }, {
+                name: 'admins',
+                admin: true
+            }];
 
-            var tmp = {
-                dn: dn.toString(),
-                attributes: {
-                    objectclass: ['group'],
-                    cn: 'admin',
-                    memberuid: result.filter(function (entry) { return entry.admin; }).map(function(entry) { return entry.id; })
+            groups.forEach(function (group) {
+                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+                var tmp = {
+                    dn: dn.toString(),
+                    attributes: {
+                        objectclass: ['group'],
+                        cn: group.name,
+                        memberuid: members.map(function(entry) { return entry.id; })
+                    }
+                };
+
+                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
+                    res.send(tmp);
+                    debug('ldap group send:', tmp);
                 }
-            };
-
-            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
-                res.send(tmp);
-                debug('ldap group send:', tmp);
-            }
+            });
 
             res.end();
         });

src/locker.js

@@ -9,6 +9,7 @@ function Locker() {
     this._operation = null;
     this._timestamp = null;
     this._watcherId = -1;
+    this._lockDepth = 0; // recursive locks
 }
 util.inherits(Locker, EventEmitter);
 
@@ -24,6 +25,7 @@ Locker.prototype.lock = function (operation) {
     if (this._operation !== null) return new Error('Already locked for ' + this._operation);
 
     this._operation = operation;
+    ++this._lockDepth;
     this._timestamp = new Date();
     var that = this;
     this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
@@ -35,21 +37,35 @@ Locker.prototype.lock = function (operation) {
     return null;
 };
 
+Locker.prototype.recursiveLock = function (operation) {
+    if (this._operation === operation) {
+        ++this._lockDepth;
+        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
+        return null;
+    }
+
+    return this.lock(operation);
+};
+
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
-    debug('Released : %s', this._operation);
+    if (--this._lockDepth === 0) {
+        debug('Released : %s', this._operation);
 
-    this._operation = null;
-    this._timestamp = null;
-    clearInterval(this._watcherId);
-    this._watcherId = -1;
+        this._operation = null;
+        this._timestamp = null;
+        clearInterval(this._watcherId);
+        this._watcherId = -1;
+    } else {
+        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
+    }
 
     this.emit('unlocked', operation);
 
     return null;
-}
+};
 
 exports = module.exports = new Locker();

src/mail_templates/feedback.ejs

@@ -0,0 +1,15 @@
+<%if (format === 'text') { %>
+
+New <%= type %> from <%= fqdn %>.
+
+Sender:  <%= user.email %>
+Sent at: <%= new Date().toUTCString() %>
+
+Subject: <%= subject %>
+-----------------------------------------------------------
+<%= description %>
+
+<% } else { %>
+
+<% } %>
+

src/mailer.js

@@ -15,7 +15,12 @@ exports = module.exports = {
 
     sendCrashNotification: sendCrashNotification,
 
-    appDied: appDied
+    appDied: appDied,
+
+    FEEDBACK_TYPE_FEEDBACK: 'feedback',
+    FEEDBACK_TYPE_TICKET: 'ticket',
+    FEEDBACK_TYPE_APP: 'app',
+    sendFeedback: sendFeedback
 };
 
 var assert = require('assert'),
@@ -277,3 +282,21 @@ function sendCrashNotification(program, context) {
 
     enqueue(mailOptions);
 }
+
+function sendFeedback(user, type, subject, description) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof subject, 'string');
+    assert.strictEqual(typeof description, 'string');
+
+    assert(type === exports.FEEDBACK_TYPE_TICKET || type === exports.FEEDBACK_TYPE_FEEDBACK || type === exports.FEEDBACK_TYPE_APP);
+
+    var mailOptions = {
+        from: config.get('adminEmail'),
+        to: 'support@cloudron.io',
+        subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
+        text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})
+    };
+
+    enqueue(mailOptions);
+}

src/oauth2views/header.ejs

@@ -25,4 +25,4 @@
 
 </head>
 
-<body>
+<body class="oauth">

src/oauth2views/login.ejs

@@ -1,32 +1,42 @@
 <% include header %>
 
-<center>
-    <h1>Login to <%= applicationName %></h1>
-</center>
-
-<% if (error) { %>
-<center>
-    <br/><br/>
-    <h4 class="has-error"><%= error %></h4>
-</center>
-<% } %>
-
 <div class="container">
     <div class="row">
         <div class="col-md-6 col-md-offset-3">
-            <form id="loginForm" action="" method="post">
-                <input type="hidden" name="_csrf" value="<%= csrf %>"/>
-                <div class="form-group">
-                    <label class="control-label" for="inputUsername">Username or Email</label>
-                    <input type="text" class="form-control" id="inputUsername" name="username" autofocus required>
+            <div class="card">
+                <div class="row">
+                    <div class="col-md-12" style="text-align: center;">
+                        <img width="128" height="128" src="<%= applicationLogo %>"/>
+                        <h1>Login to <%= applicationName %> on <%= cloudronName %></h1>
+                        <br/>
+                    </div>
                 </div>
-                <div class="form-group">
-                    <label class="control-label" for="inputPassword">Password</label>
-                    <input type="password" class="form-control" name="password" id="inputPassword" required>
+                <br/>
+            <% if (error) { %>
+                <div class="row">
+                    <div class="col-md-12">
+                        <h4 class="has-error"><%= error %></h4>
+                    </div>
                 </div>
-                <input class="btn btn-primary btn-outline pull-right" type="submit" value="Sign in"/>
-            </form>
-            <a href="/api/v1/session/password/resetRequest.html">Reset your password</a>
+            <% } %>
+                <div class="row">
+                    <div class="col-md-12">
+                        <form id="loginForm" action="" method="post">
+                            <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+                            <div class="form-group">
+                                <label class="control-label" for="inputUsername">Username or Email</label>
+                                <input type="text" class="form-control" id="inputUsername" name="username" autofocus required>
+                            </div>
+                            <div class="form-group">
+                                <label class="control-label" for="inputPassword">Password</label>
+                                <input type="password" class="form-control" name="password" id="inputPassword" required>
+                            </div>
+                            <input class="btn btn-primary btn-outline pull-right" type="submit" value="Sign in"/>
+                        </form>
+                        <a href="/api/v1/session/password/resetRequest.html">Reset your password</a>
+                    </div>
+                </div>
+            </div>
         </div>
     </div>
 </div>
@@ -34,6 +44,8 @@
 <script>
 
 (function () {
+    'use strict';
+
     var search = window.location.search.slice(1).split('&').map(function (item) { return item.split('='); }).reduce(function (o, k) { o[k[0]] = k[1]; return o; }, {});
 
     document.getElementById('loginForm').action = '/api/v1/session/login?returnTo=' + search.returnTo;

src/progress.js

@@ -21,6 +21,7 @@ var progress = {
     backup: null
 };
 
+// We use -1 for percentage to indicate errors
 function set(tag, percent, message) {
     assert(tag === exports.UPDATE || tag === exports.BACKUP);
     assert.strictEqual(typeof percent, 'number');

src/routes/apps.js

@@ -117,7 +117,7 @@ function installApp(req, res, next) {
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
 
     // allow tests to provide an appId for testing
-    var appId = (process.env.NODE_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
+    var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
 
     debug('Installing app id:%s storeid:%s loc:%s port:%j restrict:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.manifest);
 

src/routes/cloudron.js

@@ -11,13 +11,15 @@ exports = module.exports = {
     getConfig: getConfig,
     update: update,
     migrate: migrate,
-    setCertificate: setCertificate
+    setCertificate: setCertificate,
+    feedback: feedback
 };
 
 var assert = require('assert'),
     cloudron = require('../cloudron.js'),
     config = require('../config.js'),
     progress = require('../progress.js'),
+    mailer = require('../mailer.js'),
     CloudronError = cloudron.CloudronError,
     debug = require('debug')('box:routes/cloudron'),
     HttpError = require('connect-lastmile').HttpError,
@@ -157,3 +159,15 @@ function setCertificate(req, res, next) {
         next(new HttpSuccess(202, {}));
     });
 }
+
+function feedback(req, res, next) {
+    assert.strictEqual(typeof req.user, 'object');
+
+    if (req.body.type !== mailer.FEEDBACK_TYPE_FEEDBACK && req.body.type !== mailer.FEEDBACK_TYPE_TICKET && req.body.type !== mailer.FEEDBACK_TYPE_APP) return next(new HttpError(400, 'type must be either "ticket", "feedback" or "app"'));
+    if (typeof req.body.subject !== 'string' || !req.body.subject) return next(new HttpError(400, 'subject must be string'));
+    if (typeof req.body.description !== 'string' || !req.body.description) return next(new HttpError(400, 'description must be string'));
+
+    mailer.sendFeedback(req.user, req.body.type, req.body.subject, req.body.description);
+
+    next(new HttpSuccess(201, {}));
+}

src/routes/developer.js

@@ -6,7 +6,8 @@ exports = module.exports = {
     enabled: enabled,
     setEnabled: setEnabled,
     status: status,
-    login: login
+    login: login,
+    apps: apps
 };
 
 var developer = require('../developer.js'),
@@ -46,3 +47,10 @@ function login(req, res, next) {
         });
   })(req, res, next);
 }
+
+function apps(req, res, next) {
+    developer.getNonApprovedApps(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+        next(new HttpSuccess(200, { apps: result }));
+    });
+}

src/routes/oauth2.js

@@ -16,6 +16,7 @@ var assert = require('assert'),
     querystring = require('querystring'),
     util = require('util'),
     session = require('connect-ensure-login'),
+    settings = require('../settings.js'),
     tokendb = require('../tokendb'),
     appdb = require('../appdb'),
     url = require('url'),
@@ -170,6 +171,8 @@ function sendErrorPageOrRedirect(req, res, message) {
     }
 }
 
+// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
+// This usually happens when the OAuth client ID is wrong
 function sendError(req, res, message) {
     assert.strictEqual(typeof req, 'object');
     assert.strictEqual(typeof res, 'object');
@@ -188,37 +191,47 @@ function loginForm(req, res) {
     var u = url.parse(req.session.returnTo, true);
     if (!u.query.client_id) return sendErrorPageOrRedirect(req, res, 'Invalid login request. No client_id provided.');
 
-    function render(applicationName) {
+    var cloudronName = '';
+
+    function render(applicationName, applicationLogo) {
         res.render('login', {
             adminOrigin: config.adminOrigin(),
             csrf: req.csrfToken(),
+            cloudronName: cloudronName,
             applicationName: applicationName,
+            applicationLogo: applicationLogo,
             error: req.query.error || null
         });
     }
 
-    clientdb.get(u.query.client_id, function (error, result) {
-        if (error) return sendError(req, res, 'Unknown OAuth client');
+    settings.getCloudronName(function (error, name) {
+        if (error) return sendError(req, res, 'Internal Error');
 
-        // Handle our different types of oauth clients
-        var appId = result.appId;
-        if (appId === constants.ADMIN_CLIENT_ID) {
-            return render(constants.ADMIN_NAME);
-        } else if (appId === constants.TEST_CLIENT_ID) {
-            return render(constants.TEST_NAME);
-        } else if (appId.indexOf('external-') === 0) {
-            return render('External Application');
-        } else if (appId.indexOf('addon-') === 0) {
-            appId = appId.slice('addon-'.length);
-        } else if (appId.indexOf('proxy-') === 0) {
-            appId = appId.slice('proxy-'.length);
-        }
+        cloudronName = name;
 
-        appdb.get(appId, function (error, result) {
-            if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
+        clientdb.get(u.query.client_id, function (error, result) {
+            if (error) return sendError(req, res, 'Unknown OAuth client');
 
-            var applicationName = result.location || config.fqdn();
-            render(applicationName);
+            // Handle our different types of oauth clients
+            var appId = result.appId;
+            if (appId === constants.ADMIN_CLIENT_ID) {
+                return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
+            } else if (appId === constants.TEST_CLIENT_ID) {
+                return render(constants.TEST_NAME, '/api/v1/cloudron/avatar');
+            } else if (appId.indexOf('external-') === 0) {
+                return render('External Application', '/api/v1/cloudron/avatar');
+            } else if (appId.indexOf('addon-') === 0) {
+                appId = appId.slice('addon-'.length);
+            } else if (appId.indexOf('proxy-') === 0) {
+                appId = appId.slice('proxy-'.length);
+            }
+
+            appdb.get(appId, function (error, result) {
+                if (error) return sendErrorPageOrRedirect(req, res, 'Unknown Application for those OAuth credentials');
+
+                var applicationName = result.location || config.fqdn();
+                render(applicationName, '/api/v1/cloudron/avatar');
+            });
         });
     });
 }

src/routes/test/apps-test.js

@@ -573,7 +573,8 @@ describe('App installation', function () {
         docker.getContainer(appEntry.containerId).inspect(function (error, data) {
             expect(error).to.not.be.ok();
             expect(data.Config.ExposedPorts['7777/tcp']).to.eql({ });
-            expect(data.Config.Env).to.contain('ADMIN_ORIGIN=' + config.adminOrigin());
+            expect(data.Config.Env).to.contain('WEBADMIN_ORIGIN=' + config.adminOrigin());
+            expect(data.Config.Env).to.contain('API_ORIGIN=' + config.adminOrigin());
             expect(data.Config.Env).to.contain('CLOUDRON=1');
             clientdb.getByAppId('addon-' + appResult.id, function (error, client) {
                 expect(error).to.not.be.ok();

src/routes/test/backups-test.js

@@ -50,7 +50,7 @@ function setup(done) {
         },
 
         function addApp(callback) {
-            var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
+            var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
             appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, '' /* accessRestriction */, callback);
         }
     ], done);
@@ -119,8 +119,8 @@ describe('Backups API', function () {
 
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
-                        .put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] })
-                        .reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                        .get('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                        .reply(201, { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', sessionToken: 'sessionToken' });
 
             request.post(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })

src/routes/test/cloudron-test.js

@@ -26,6 +26,7 @@ var token = null; // authentication token
 
 var server;
 function setup(done) {
+    nock.cleanAll();
     config.set('version', '0.5.0');
     server.start(done);
 }
@@ -449,8 +450,18 @@ describe('Cloudron', function () {
         });
 
         it('fails when in wrong state', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(409, {});
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+            var scope2 = nock(config.apiServerOrigin())
+                    .get('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .reply(201, { credentials: { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', sessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .filteringRequestBody(function () { return false; })
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN')
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .filteringRequestBody(function () { return false; })
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { }).reply(409, {});
 
             injectShellMock();
 
@@ -462,7 +473,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }
@@ -501,6 +512,158 @@ describe('Cloudron', function () {
             });
         });
     });
+
+    describe('feedback', function () {
+        before(function (done) {
+            async.series([
+                setup,
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    config._reset();
+
+                    request.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(error).to.not.be.ok();
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        // stash token for further use
+                        token = result.body.token;
+
+                        callback();
+                    });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails without token', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'ticket', subject: 'some subject', description: 'some description' })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(401);
+                done();
+            });
+        });
+
+        it('fails without type', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ subject: 'some subject', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails with empty type', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: '', subject: 'some subject', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails with unknown type', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'foobar', subject: 'some subject', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds with ticket type', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'ticket', subject: 'some subject', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(201);
+                done();
+            });
+        });
+
+        it('succeeds with app type', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'app', subject: 'some subject', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(201);
+                done();
+            });
+        });
+
+        it('fails without description', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'ticket', subject: 'some subject' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails with empty subject', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'ticket', subject: '', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('fails with empty description', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'ticket', subject: 'some subject', description: '' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds with feedback type', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'feedback', subject: 'some subject', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(201);
+                done();
+            });
+        });
+
+        it('fails without subject', function (done) {
+            request.post(SERVER_URL + '/api/v1/cloudron/feedback')
+                   .send({ type: 'ticket', description: 'some description' })
+                   .query({ access_token: token })
+                   .end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+    });
 });
 
 

src/routes/test/start_addons.sh

@@ -2,6 +2,10 @@
 
 set -eu -o pipefail
 
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+
+source ${SOURCE_DIR}/setup/INFRA_VERSION
+
 readonly mysqldatadir="/tmp/mysqldata-$(date +%s)"
 readonly postgresqldatadir="/tmp/postgresqldata-$(date +%s)"
 readonly mongodbdatadir="/tmp/mongodbdata-$(date +%s)"
@@ -20,7 +24,7 @@ start_postgresql() {
 
     docker rm -f postgresql 2>/dev/null 1>&2 || true
 
-    docker run -dtP --name=postgresql -v "${postgresqldatadir}:/var/lib/postgresql" -v /tmp/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh cloudron/postgresql:0.3.0 >/dev/null
+    docker run -dtP --name=postgresql -v "${postgresqldatadir}:/var/lib/postgresql" -v /tmp/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh "${POSTGRESQL_IMAGE}" >/dev/null
 }
 
 start_mysql() {
@@ -36,7 +40,7 @@ start_mysql() {
 
     docker rm -f mysql 2>/dev/null 1>&2 || true
 
-    docker run -dP --name=mysql -v "${mysqldatadir}:/var/lib/mysql" -v /tmp/mysql_vars.sh:/etc/mysql/mysql_vars.sh cloudron/mysql:0.3.0 >/dev/null
+    docker run -dP --name=mysql -v "${mysqldatadir}:/var/lib/mysql" -v /tmp/mysql_vars.sh:/etc/mysql/mysql_vars.sh "${MYSQL_IMAGE}" >/dev/null
 }
 
 start_mongodb() {
@@ -52,7 +56,7 @@ start_mongodb() {
 
     docker rm -f mongodb 2>/dev/null 1>&2 || true
 
-    docker run -dP --name=mongodb -v "${mongodbdatadir}:/var/lib/mongodb" -v /tmp/mongodb_vars.sh:/etc/mongodb_vars.sh cloudron/mongodb:0.3.0 >/dev/null
+    docker run -dP --name=mongodb -v "${mongodbdatadir}:/var/lib/mongodb" -v /tmp/mongodb_vars.sh:/etc/mongodb_vars.sh "${MONGODB_IMAGE}" >/dev/null
 }
 
 start_mysql

src/scripts/backupapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: backup.sh <appid> <url> <key>"
+    echo "Usage: backupapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -22,6 +22,7 @@ readonly DATA_DIR="${HOME}/data"
 app_id="$1"
 backup_url="$2"
 backup_key="$3"
+session_token="$4"
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
@@ -31,9 +32,17 @@ btrfs subvolume snapshot -r "${app_data_dir}" "${app_data_snapshot}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${app_data_snapshot}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT "${headers[@]}" --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/backupbox.sh

@@ -13,12 +13,13 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 2 ]; then
-    echo "Usage: backupbox.sh <url> <key>"
+    echo "Usage: backupbox.sh <url> <key> [aws session token]"
     exit 1
 fi
 
 backup_url="$1"
 backup_key="$2"
+session_token="$3"
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"
@@ -32,9 +33,17 @@ btrfs subvolume snapshot -r "${BOX_DATA_DIR}" "${box_snapshot_dir}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${box_snapshot_dir}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/collectlogs.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if [[ $EUID -ne 0 ]]; then
+    echo "This script should be run as root." >&2
+    exit 1
+fi
+
+if [[ $# == 1 && "$1" == "--check" ]]; then
+    echo "OK"
+    exit 0
+fi
+
+if [ $# -lt 1 ]; then
+    echo "Usage: collectlogs.sh <program>"
+    exit 1
+fi
+
+readonly program_name=$1
+
+echo "${program_name}.log"
+echo "-------------------"
+journalctl --no-pager -u ${program_name} -n 100
+echo
+echo
+echo "dmesg"
+echo "-----"
+dmesg | tail --lines=100
+echo
+echo
+echo "docker"
+echo "------"
+journalctl --no-pager -u docker -n 50
+echo
+echo
+
+
+

src/scripts/createappdir.sh

@@ -17,7 +17,7 @@ if [[ "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${NODE_ENV}" == "cloudron" ]]; then
+if [[ "${BOX_ENV}" == "cloudron" ]]; then
     readonly app_data_dir="${HOME}/data/$1"
     btrfs subvolume create "${app_data_dir}"
     mkdir -p "${app_data_dir}/data"

src/scripts/reboot.sh

@@ -12,7 +12,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${NODE_ENV}" == "cloudron" ]]; then
+if [[ "${BOX_ENV}" == "cloudron" ]]; then
     shutdown -r now
 fi
 

src/scripts/reloadcollectd.sh

@@ -12,7 +12,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${NODE_ENV}" == "cloudron" ]]; then
+if [[ "${BOX_ENV}" == "cloudron" ]]; then
     /etc/init.d/collectd restart
 fi
 

src/scripts/reloadnginx.sh

@@ -12,12 +12,6 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${OSTYPE}" == "darwin"* ]]; then
-    # On Mac, brew installs supervisor in /usr/local/bin
-    export PATH=$PATH:/usr/local/bin
-fi
-
-if [[ "${NODE_ENV}" == "cloudron" ]]; then
+if [[ "${BOX_ENV}" == "cloudron" ]]; then
     nginx -s reload
 fi
-

src/scripts/restoreapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: restoreapp.sh <appid> <url> <key>"
+    echo "Usage: restoreapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -23,6 +23,7 @@ readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max
 app_id="$1"
 restore_url="$2"
 restore_key="$3"
+session_token="$4"
 
 echo "Downloading backup: ${restore_url} and key: ${restore_key}"
 
@@ -30,7 +31,14 @@ for try in `seq 1 5`; do
     echo "Download backup from ${restore_url} (try ${try})"
     error_log=$(mktemp)
 
-    if $curl -L "${restore_url}" \
+    headers=("") # empty element required (http://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u)
+
+    # federated tokens in CaaS case need session token
+    if [[ ! -z "${session_token}" ]]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
+    if $curl -L "${headers[@]}" "${restore_url}" \
         | openssl aes-256-cbc -d -pass "pass:${restore_key}" \
         | tar -zxf - -C "${DATA_DIR}/${app_id}" 2>"${error_log}"; then
         chown -R yellowtent:yellowtent "${DATA_DIR}/${app_id}"

src/scripts/rmappdir.sh

@@ -17,7 +17,7 @@ if [[ "$1" == "--check" ]]; then
     exit 0
 fi
 
-if [[ "${NODE_ENV}" == "cloudron" ]]; then
+if [[ "${BOX_ENV}" == "cloudron" ]]; then
     readonly app_data_dir="${HOME}/data/$1"
     if [[ -d "${app_data_dir}" ]]; then
         find "${app_data_dir}" -mindepth 1 -delete

src/server.js

@@ -43,7 +43,7 @@ function initializeExpressSync() {
     app.set('view options', { layout: true, debug: true });
     app.set('view engine', 'ejs');
 
-    if (process.env.NODE_ENV === 'test') {
+    if (process.env.BOX_ENV === 'test') {
        app.use(express.static(path.join(__dirname, '/../webadmin')));
     } else {
         app.use(middleware.morgan('dev', { immediate: false }));
@@ -92,15 +92,19 @@ function initializeExpressSync() {
     router.post('/api/v1/developer', developerScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.developer.setEnabled);
     router.get ('/api/v1/developer', developerScope, routes.developer.enabled, routes.developer.status);
     router.post('/api/v1/developer/login', routes.developer.enabled, routes.developer.login);
+    router.get ('/api/v1/developer/apps', developerScope, routes.developer.enabled, routes.developer.apps);
 
     // private routes
     router.get ('/api/v1/cloudron/config', rootScope, routes.cloudron.getConfig);
     router.post('/api/v1/cloudron/update', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
-    router.get ('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
+    router.post('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
     router.post('/api/v1/cloudron/migrate', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.migrate);
     router.post('/api/v1/cloudron/certificate', rootScope, multipart, routes.cloudron.setCertificate);
     router.get ('/api/v1/cloudron/graphs', rootScope, routes.graphs.getGraphs);
 
+    // feedback
+    router.post('/api/v1/cloudron/feedback', usersScope, routes.cloudron.feedback);
+
     router.get ('/api/v1/profile', profileScope, routes.user.profile);
 
     router.get ('/api/v1/users', usersScope, routes.user.list);

src/subdomainerror.js

@@ -0,0 +1,39 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    util = require('util');
+
+exports = module.exports = SubdomainError;
+
+function SubdomainError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(SubdomainError, Error);
+
+SubdomainError.NOT_FOUND = 'No such domain';
+SubdomainError.INTERNAL_ERROR = 'Internal error';
+SubdomainError.EXTERNAL_ERROR = 'External error';
+SubdomainError.STILL_BUSY = 'Still busy';
+SubdomainError.FAILED_TOO_OFTEN = 'Failed too often';
+SubdomainError.ALREADY_EXISTS = 'Domain already exists';
+SubdomainError.BAD_FIELD = 'Bad Field';
+SubdomainError.BAD_STATE = 'Bad State';
+SubdomainError.INVALID_ZONE_NAME = 'Invalid domain name';
+SubdomainError.INVALID_TASK = 'Invalid task';

src/subdomains.js

@@ -0,0 +1,82 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    aws = require('./aws.js'),
+    config = require('./config.js'),
+    debug = require('debug')('box:subdomains'),
+    util = require('util'),
+    SubdomainError = require('./subdomainerror.js');
+
+module.exports = exports = {
+    add: add,
+    addMany: addMany,
+    remove: remove,
+    status: status
+};
+
+function add(record, callback) {
+    assert.strictEqual(typeof record, 'object');
+    assert.strictEqual(typeof record.subdomain, 'string');
+    assert.strictEqual(typeof record.type, 'string');
+    assert.strictEqual(typeof record.value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: ', record);
+
+    aws.addSubdomain(config.zoneName(), record.subdomain, record.type, record.value, function (error, changeId) {
+        if (error) return callback(error);
+        callback(null, changeId);
+    });
+}
+
+function addMany(records, callback) {
+    assert(util.isArray(records));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addMany: ', records);
+
+    var changeIds = [];
+
+    async.eachSeries(records, function (record, callback) {
+        add(record, function (error, changeId) {
+            if (error) return callback(error);
+
+            changeIds.push(changeId);
+
+            callback(null);
+        });
+    }, function (error) {
+        if (error) return callback(error);
+        callback(null, changeIds);
+    });
+}
+
+function remove(record, callback) {
+    assert.strictEqual(typeof record, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('remove: ', record);
+
+    aws.delSubdomain(config.zoneName(), record.subdomain, record.type, record.value, function (error) {
+        if (error && error.reason !== SubdomainError.NOT_FOUND) return callback(error);
+
+        debug('deleteSubdomain: successfully deleted subdomain from aws.');
+
+        callback(null);
+    });
+}
+
+function status(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('status: ', changeId);
+
+    aws.getChangeStatus(changeId, function (error, status) {
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error));
+        callback(null, status === 'INSYNC' ? 'done' : 'pending');
+    });
+}

src/taskmanager.js

@@ -17,10 +17,7 @@ var appdb = require('./appdb.js'),
 var gActiveTasks = { };
 var gPendingTasks = [ ];
 
-// Task concurrency is 1 for two reasons:
-// 1. The backup scripts (app and box) turn off swap after finish disregarding other backup processes
-// 2. apptask getFreePort has race with multiprocess
-var TASK_CONCURRENCY = 1;
+var TASK_CONCURRENCY = 5;
 var NOOP_CALLBACK = function (error) { console.error(error); };
 
 function initialize(callback) {
@@ -54,7 +51,8 @@ function uninitialize(callback) {
 
 function startNextTask() {
     if (gPendingTasks.length === 0) return;
-    assert(Object.keys(gActiveTasks).length === 0); // since we allow only one task at a time
+
+    assert(Object.keys(gActiveTasks).length < TASK_CONCURRENCY);
 
     startAppTask(gPendingTasks.shift());
 }
@@ -63,7 +61,7 @@ function startAppTask(appId) {
     assert.strictEqual(typeof appId, 'string');
     assert(!(appId in gActiveTasks));
 
-    var lockError = locker.lock(locker.OP_APPTASK);
+    var lockError = locker.recursiveLock(locker.OP_APPTASK);
 
     if (lockError || Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
         debug('Reached concurrency limit, queueing task for %s', appId);

src/test/checkInstall

@@ -2,21 +2,24 @@
 
 set -eu
 
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+source ${SOURCE_DIR}/setup/INFRA_VERSION
 
 # reset sudo timestamp to avoid wrong success
 sudo -k || sudo --reset-timestamp
 
 # checks if all scripts are sudo access
-scripts=("${SOURCE_DIR}/scripts/rmappdir.sh" \
-         "${SOURCE_DIR}/scripts/createappdir.sh" \
-         "${SOURCE_DIR}/scripts/reloadnginx.sh" \
-         "${SOURCE_DIR}/scripts/backupbox.sh" \
-         "${SOURCE_DIR}/scripts/backupapp.sh" \
-         "${SOURCE_DIR}/scripts/restoreapp.sh" \
-         "${SOURCE_DIR}/scripts/reboot.sh" \
-         "${SOURCE_DIR}/scripts/backupswap.sh" \
-         "${SOURCE_DIR}/scripts/reloadcollectd.sh")
+scripts=("${SOURCE_DIR}/src/scripts/rmappdir.sh" \
+         "${SOURCE_DIR}/src/scripts/createappdir.sh" \
+         "${SOURCE_DIR}/src/scripts/reloadnginx.sh" \
+         "${SOURCE_DIR}/src/scripts/backupbox.sh" \
+         "${SOURCE_DIR}/src/scripts/backupapp.sh" \
+         "${SOURCE_DIR}/src/scripts/restoreapp.sh" \
+         "${SOURCE_DIR}/src/scripts/reboot.sh" \
+         "${SOURCE_DIR}/src/scripts/backupswap.sh" \
+         "${SOURCE_DIR}/src/scripts/collectlogs.sh" \
+         "${SOURCE_DIR}/src/scripts/reloadcollectd.sh")
 
 for script in "${scripts[@]}"; do
     if [[ $(sudo -n "${script}" --check 2>/dev/null) != "OK" ]]; then
@@ -24,7 +27,7 @@ for script in "${scripts[@]}"; do
         echo "${script} does not have sudo access."
         echo "You have to add the lines below to /etc/sudoers.d/yellowtent."
         echo ""
-        echo "Defaults!${script} env_keep=\"HOME NODE_ENV\""
+        echo "Defaults!${script} env_keep=\"HOME BOX_ENV\""
         echo "${USER} ALL=(ALL) NOPASSWD: ${script}"
         echo ""
         exit 1
@@ -36,23 +39,23 @@ if ! docker inspect girish/test:0.2.0 >/dev/null 2>/dev/null; then
     exit 1
 fi
 
-if ! docker inspect cloudron/redis:0.3.0 >/dev/null 2>/dev/null; then
-    echo "docker pull cloudron/redis:0.3.0 for tests to run"
+if ! docker inspect "${REDIS_IMAGE}" >/dev/null 2>/dev/null; then
+    echo "docker pull ${REDIS_IMAGE} for tests to run"
     exit 1
 fi
 
-if ! docker inspect cloudron/mysql:0.3.0 >/dev/null 2>/dev/null; then
-    echo "docker pull cloudron/mysql:0.3.0 for tests to run"
+if ! docker inspect "${MYSQL_IMAGE}" >/dev/null 2>/dev/null; then
+    echo "docker pull ${MYSQL_IMAGE} for tests to run"
     exit 1
 fi
 
-if ! docker inspect cloudron/postgresql:0.3.0 >/dev/null 2>/dev/null; then
-    echo "docker pull cloudron/postgresql:0.3.0 for tests to run"
+if ! docker inspect "${POSTGRESQL_IMAGE}" >/dev/null 2>/dev/null; then
+    echo "docker pull ${POSTGRESQL_IMAGE} for tests to run"
     exit 1
 fi
 
-if ! docker inspect cloudron/mongodb:0.3.0 >/dev/null 2>/dev/null; then
-    echo "docker pull cloudron/mongodb:0.3.0 for tests to run"
+if ! docker inspect "${MONGODB_IMAGE}" >/dev/null 2>/dev/null; then
+    echo "docker pull ${MONGODB_IMAGE} for tests to run"
     exit 1
 fi
 

src/test/ldap-test.js

@@ -0,0 +1,263 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+require('supererror', { splatchError: true});
+
+var database = require('../database.js'),
+    expect = require('expect.js'),
+    EventEmitter = require('events').EventEmitter,
+    async = require('async'),
+    user = require('../user.js'),
+    config = require('../config.js'),
+    ldapServer = require('../ldap.js'),
+    ldap = require('ldapjs');
+
+var USER_0 = {
+    username: 'foobar0',
+    password: 'password0',
+    email: 'foo0@bar.com'
+};
+
+var USER_1 = {
+    username: 'foobar1',
+    password: 'password1',
+    email: 'foo1@bar.com'
+};
+
+function setup(done) {
+    async.series([
+        database.initialize.bind(null),
+        database._clear.bind(null),
+        ldapServer.start.bind(null),
+        user.create.bind(null, USER_0.username, USER_0.password, USER_0.email, true, null),
+        user.create.bind(null, USER_1.username, USER_1.password, USER_1.email, false, USER_0)
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(done);
+}
+
+describe('Ldap', function () {
+    before(setup);
+    after(cleanup);
+
+    describe('bind', function () {
+        it('fails for nonexisting user', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            client.bind('cn=doesnotexist,ou=users,dc=cloudron', 'password', function (error) {
+                expect(error).to.be.a(ldap.NoSuchObjectError);
+                done();
+            });
+        });
+
+        it('fails with wrong password', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            client.bind('cn=' + USER_0.username + ',ou=users,dc=cloudron', 'wrongpassword', function (error) {
+                expect(error).to.be.a(ldap.InvalidCredentialsError);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            client.bind('cn=' + USER_0.username + ',ou=users,dc=cloudron', USER_0.password, function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+    });
+
+    describe('search users', function () {
+        it ('fails for non existing tree', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: '(&(l=Seattle)(email=*@foo.com))'
+            };
+
+            client.search('o=example', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                result.on('error', function (error) {
+                    expect(error).to.be.a(ldap.NoSuchObjectError);
+                    done();
+                });
+                result.on('end', function (result) {
+                    done(new Error('Should not succeed. Status ' + result.status));
+                });
+            });
+        });
+
+        it ('succeeds with basic filter', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: 'objectcategory=person'
+            };
+
+            client.search('ou=users,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+                    expect(entries[0].username).to.equal(USER_0.username);
+                    expect(entries[1].username).to.equal(USER_1.username);
+                    done();
+                });
+            });
+        });
+
+        it ('succeeds with username wildcard filter', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: '&(objectcategory=person)(username=foobar*)'
+            };
+
+            client.search('ou=users,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+                    expect(entries[0].username).to.equal(USER_0.username);
+                    expect(entries[1].username).to.equal(USER_1.username);
+                    done();
+                });
+            });
+        });
+
+        it ('succeeds with username filter', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: '&(objectcategory=person)(username=' + USER_0.username + ')'
+            };
+
+            client.search('ou=users,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(1);
+                    expect(entries[0].username).to.equal(USER_0.username);
+                    expect(entries[0].memberof.length).to.equal(2);
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('search groups', function () {
+        it ('succeeds with basic filter', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: 'objectclass=group'
+            };
+
+            client.search('ou=groups,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+                    expect(entries[0].cn).to.equal('users');
+                    expect(entries[0].memberuid.length).to.equal(2);
+                    expect(entries[0].memberuid[0]).to.equal(USER_0.username);
+                    expect(entries[0].memberuid[1]).to.equal(USER_1.username);
+                    expect(entries[1].cn).to.equal('admins');
+                    // if only one entry, the array becomes a string :-/
+                    expect(entries[1].memberuid).to.equal(USER_0.username);
+                    done();
+                });
+            });
+        });
+
+        it ('succeeds with cn wildcard filter', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: '&(objectclass=group)(cn=*)'
+            };
+
+            client.search('ou=groups,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+                    expect(entries[0].cn).to.equal('users');
+                    expect(entries[0].memberuid.length).to.equal(2);
+                    expect(entries[0].memberuid[0]).to.equal(USER_0.username);
+                    expect(entries[0].memberuid[1]).to.equal(USER_1.username);
+                    expect(entries[1].cn).to.equal('admins');
+                    // if only one entry, the array becomes a string :-/
+                    expect(entries[1].memberuid).to.equal(USER_0.username);
+                    done();
+                });
+            });
+        });
+
+        it('succeeds with memberuid filter', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: '&(objectclass=group)(memberuid=' + USER_1.username + ')'
+            };
+
+            client.search('ou=groups,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(1);
+                    expect(entries[0].cn).to.equal('users');
+                    expect(entries[0].memberuid.length).to.equal(2);
+                    done();
+                });
+            });
+        });
+    });
+});

src/webhooks.js

@@ -0,0 +1,47 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    backupDone: backupDone
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:webhooks'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function backupDone(filename, app, appBackupIds, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert(!app || typeof app === 'object');
+    assert(!appBackupIds || util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('backupDone():', filename);
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupDone';
+        var data = {
+            boxVersion: config.version(),
+            restoreKey: filename,
+            appId: app ? app.id : null,
+            appVersion: app ? app.manifest.version : null,
+            appBackupIds: appBackupIds
+        };
+
+        superagent.post(url).send(data).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 200) return callback(new Error(result.text));
+            if (!result.body) return callback(new Error('Unexpected response'));
+
+            debug('backupDone()', filename);
+
+            return callback(null);
+        });
+    } else {
+        // TODO call custom webhook
+        callback(null);
+    }
+}

webadmin/src/appstatus.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron App Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 
@@ -30,20 +30,14 @@
         // create main application module
         var app = angular.module('Application', []);
 
-        // FIXME this does not work with custom domains!
-        function detectApiOrigin() {
-            var host = window.location.host;
-            var tmp = host.split('.')[0];
-            if (tmp.indexOf('-') === -1) return 'https://my-' + host;
-            else return 'https://my' + tmp.slice(tmp.indexOf('-')) + host.slice(tmp.length);
-        }
-
         app.controller('Controller', ['$scope', '$http', function ($scope, $http) {
-            $scope.apiOrigin = detectApiOrigin();
+            var search = decodeURIComponent(window.location.search).slice(1).split('&').map(function (item) { return item.split('='); }).reduce(function (o, k) { o[k[0]] = k[1]; return o; }, {});
+
             $scope.cloudronName = 'Cloudron';
+            $scope.referrer = search.referrer || null;
 
             // try to fetch cloudron status
-            $http.get($scope.apiOrigin + '/api/v1/cloudron/status').success(function(data, status) {
+            $http.get('/api/v1/cloudron/status').success(function(data, status) {
                 if (status !== 200 || typeof data !== 'object') return console.error(status, data);
                 $scope.cloudronName = data.cloudronName;
                 document.title = $scope.cloudronName + ' App Error';
@@ -64,7 +58,7 @@
         <h1> {{cloudronName}} </h1>
 
         <h3> <i class="fa fa-frown-o fa-fw text-danger"></i> Something has gone wrong </h3>
-        This app is currently not running. Please retry later.
+        This app is currently not running. <a href="{{ referrer }}">Please retry later</a>.
 
         <footer>
             <span class="text-muted"><a href="mailto: support@cloudron.io">Contact Support</a> - Copyright &copy; Cloudron 2014-15</span>

webadmin/src/error.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 
@@ -30,22 +30,13 @@
         // create main application module
         var app = angular.module('Application', []);
 
-        // FIXME this does not work with custom domains!
-        function detectApiOrigin() {
-            var host = window.location.host;
-            var tmp = host.split('.')[0];
-            if (tmp.indexOf('-') === -1) return 'https://my-' + host;
-            else return 'https://my' + tmp.slice(tmp.indexOf('-')) + host.slice(tmp.length);
-        }
-
         app.controller('Controller', ['$scope', '$http', function ($scope, $http) {
-            $scope.apiOrigin = detectApiOrigin();
             $scope.cloudronName = 'Cloudron';
             $scope.webServerOriginLink = '/';
             $scope.errorMessage = '';
 
             // try to fetch at least config.json to get appstore url
-            $http.get($scope.apiOrigin + '/config.json').success(function(data, status) {
+            $http.get('/config.json').success(function(data, status) {
                 if (status !== 200 || typeof data !== 'object') return console.error(status, data);
                 $scope.webServerOriginLink = data.webServerOrigin + '/console.html';
             }).error(function (data, status) {
@@ -54,7 +45,7 @@
             });
 
             // try to fetch cloudron status
-            $http.get($scope.apiOrigin + '/api/v1/cloudron/status').success(function(data, status) {
+            $http.get('/api/v1/cloudron/status').success(function(data, status) {
                 if (status !== 200 || typeof data !== 'object') return console.error(status, data);
                 $scope.cloudronName  = data.cloudronName;
                 document.title = $scope.cloudronName + ' Error';
@@ -76,7 +67,7 @@
 
 <div class="wrapper">
     <div class="content">
-        <img src="/img/logo_inverted_192.png"/>
+        <img src="/api/v1/cloudron/avatar" onerror="this.src = '/img/logo_inverted_192.png'"/>
         <h1> {{cloudronName}} </h1>
 
         <div ng-show="errorCode == 0">

webadmin/src/index.html

@@ -6,7 +6,10 @@
 
     <title> Cloudron </title>
 
-    <link href="/img/favicon.png" rel="icon" type="image/png">
+    <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
+
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
 
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
@@ -48,9 +51,6 @@
     <!-- Main Application -->
     <script src="js/index.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body>
@@ -81,7 +81,7 @@
                         <li ng-repeat="change in config.update.box.changelog">{{change}}</li>
                     </ul>
                     <br/>
-                    <fieldset>
+                    <fieldset ng-show="installedApps | readyToUpdate">
                         <form name="update_form" role="form" ng-submit="doUpdate()" autocomplete="off">
                             <div class="form-group" ng-class="{ 'has-error': update_form.password.$dirty && update_form.password.$invalid }">
                                 <label class="control-label" for="inputUpdatePassword">Give your password to verify that you are performing that action</label>
@@ -99,7 +99,7 @@
                 </div>
                 <div class="modal-footer">
                     <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
+                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy" ng-show="installedApps | readyToUpdate"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
                 </div>
             </div>
         </div>
@@ -117,6 +117,7 @@
                         <span class="icon-bar"></span>
                         <span class="icon-bar"></span>
                     </button>
+                    <a class="navbar-brand navbar-brand-icon" href="index.html"><img src="/api/v1/cloudron/avatar" width="40" height="40"/></a>
                     <a class="navbar-brand" href="index.html">{{config.cloudronName || 'Cloudron'}}</a>
                 </div>
                 <!-- /.navbar-header -->
@@ -142,9 +143,10 @@
                             <ul class="dropdown-menu" role="menu">
                                 <li><a href="#/account"><i class="fa fa-user fa-fw"></i> Account</a></li>
                                 <li ng-show="user.admin && config.isDev"><a href="#/dns"><i class="fa fa-wrench fa-fw"></i> DNS Management</a></li>
-                                <li ng-show="user.admin"><a href="#/upgrade"><i class="fa fa-arrow-up fa-fw"></i> Upgrade</a></li>
-                                <li ng-show="user.admin"><a href="#/settings"><i class="fa fa-wrench fa-fw"></i> Settings</a></li>
                                 <li ng-show="user.admin"><a href="#/graphs"><i class="fa fa-bar-chart fa-fw"></i> Graphs</a></li>
+                                <li ng-show="user.admin"><a href="#/upgrade"><i class="fa fa-arrow-up fa-fw"></i> Upgrade</a></li>
+                                <li><a href="#/support"><i class="fa fa-comment fa-fw"></i> Support</a></li>
+                                <li ng-show="user.admin"><a href="#/settings"><i class="fa fa-wrench fa-fw"></i> Settings</a></li>
                                 <li class="divider"></li>
                                 <li><a href="" ng-click="logout($event)"><i class="fa fa-sign-out fa-fw"></i> Logout</a></li>
                             </ul>

webadmin/src/js/appstore.js

@@ -63,6 +63,17 @@ angular.module('Application').service('AppStore', ['$http', 'Client', function (
         });
     };
 
+    AppStore.prototype.getAppByIdAndVersion = function (appId, version, callback) {
+        if (Client.getConfig().apiServerOrigin === null) return callback(new AppStoreError(420, 'Enhance Your Calm'));
+
+        $http.get(Client.getConfig().apiServerOrigin + '/api/v1/apps/' + appId + '/versions/' + version).success(function (data, status) {
+            if (status !== 200) return callback(new AppStoreError(status, data));
+            return callback(null, data);
+        }).error(function (data, status) {
+            return callback(new AppStoreError(status, data));
+        });
+    };
+
     AppStore.prototype.getManifest = function (appId, callback) {
         if (Client.getConfig().apiServerOrigin === null) return callback(new AppStoreError(420, 'Enhance Your Calm'));
 

webadmin/src/js/client.js

@@ -32,7 +32,7 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
                         return callback(new ClientError(status, data));
                     }
 
-                    if (result.update) window.location.href = '/update.html';
+                    if (result.update && result.update.percent !== -1) window.location.href = '/update.html';
                     else callback(new ClientError(status, data));
                 }, function (data, status) {
                     client.error(data);
@@ -324,6 +324,15 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
         }).error(defaultErrorHandler(callback));
     };
 
+    Client.prototype.getNonApprovedApps = function (callback) {
+        if (!this._config.developerMode) return callback(null, []);
+
+        $http.get(client.apiOrigin + '/api/v1/developer/apps').success(function (data, status) {
+            if (status !== 200 || typeof data !== 'object') return callback(new ClientError(status, data));
+            callback(null, data.apps || []);
+        }).error(defaultErrorHandler(callback));
+    };
+
     Client.prototype.getApp = function (appId, callback) {
         var appFound = null;
         this._installedApps.some(function (app) {
@@ -416,7 +425,7 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
     };
 
     Client.prototype.reboot = function (callback) {
-        $http.get(client.apiOrigin + '/api/v1/cloudron/reboot').success(function(data, status) {
+        $http.post(client.apiOrigin + '/api/v1/cloudron/reboot', { }).success(function(data, status) {
             if (status !== 202 || typeof data !== 'object') return callback(new ClientError(status, data));
             callback(null, data);
         }).error(defaultErrorHandler(callback));
@@ -460,6 +469,19 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
         }).error(defaultErrorHandler(callback));
     };
 
+    Client.prototype.feedback = function (type, subject, description, callback) {
+        var data = {
+            type: type,
+            subject: subject,
+            description: description
+        };
+
+        $http.post(client.apiOrigin + '/api/v1/cloudron/feedback', data).success(function (data, status) {
+            if (status !== 201) return callback(new ClientError(status, data));
+            callback(null);
+        }).error(defaultErrorHandler(callback));
+    };
+
     Client.prototype.createUser = function (username, email, callback) {
         var data = {
             username: username,

webadmin/src/js/index.js

@@ -37,6 +37,9 @@ app.config(['$routeProvider', function ($routeProvider) {
     }).when('/settings', {
         controller: 'SettingsController',
         templateUrl: 'views/settings.html'
+    }).when('/support', {
+        controller: 'SupportController',
+        templateUrl: 'views/support.html'
     }).when('/upgrade', {
         controller: 'UpgradeController',
         templateUrl: 'views/upgrade.html'

webadmin/src/js/main.js

@@ -86,9 +86,6 @@ angular.module('Application').controller('MainController', ['$scope', '$route',
                 if (error && error.statusCode === 401) return $scope.login();
                 if (error) return $scope.error(error);
 
-                // check if we are actually updateing
-                if (Client.getConfig().progress.update) window.location.href = '/update.html';
-
                 Client.refreshUserInfo(function (error, result) {
                     if (error) return $scope.error(error);
 
@@ -122,7 +119,8 @@ angular.module('Application').controller('MainController', ['$scope', '$route',
 
     // wait till the view has loaded until showing a modal dialog
     Client.onConfig(function (config) {
-        if (config.progress.update) {
+        // check if we are actually updating
+        if (config.progress.update && config.progress.update.percent !== -1) {
             window.location.href = '/update.html';
         }
 

webadmin/src/js/setup.js

@@ -28,11 +28,8 @@ app.config(['$routeProvider', function ($routeProvider) {
         controller: 'StepController',
         templateUrl: 'views/setup/step2.html'
     }).when('/step3', {
-        controller: 'StepController',
-        templateUrl: 'views/setup/step3.html'
-    }).when('/step4', {
         controller: 'FinishController',
-        templateUrl: 'views/setup/step4.html'
+        templateUrl: 'views/setup/step3.html'
     }).otherwise({ redirectTo: '/'});
 }]);
 
@@ -51,27 +48,51 @@ app.service('Wizard', [ function () {
         }, {
             file: null,
             data: null,
-            url: '/img/avatars/cloudfacegreen.png'
+            url: '/img/avatars/rubber-duck.png'
         }, {
             file: null,
             data: null,
-            url: '/img/avatars/cloudfaceturquoise.png'
+            url: '/img/avatars/carrot.png'
         }, {
             file: null,
             data: null,
-            url: '/img/avatars/cloudglassesgreen.png'
+            url: '/img/avatars/cup.png'
         }, {
             file: null,
             data: null,
-            url: '/img/avatars/cloudglassespink.png'
+            url: '/img/avatars/football.png'
         }, {
             file: null,
             data: null,
-            url: '/img/avatars/cloudglassesturquoise.png'
+            url: '/img/avatars/owl.png'
         }, {
             file: null,
             data: null,
-            url: '/img/avatars/cloudglassesyellow.png'
+            url: '/img/avatars/space-rocket.png'
+        }, {
+            file: null,
+            data: null,
+            url: '/img/avatars/armchair.png'
+        }, {
+            file: null,
+            data: null,
+            url: '/img/avatars/cap.png'
+        }, {
+            file: null,
+            data: null,
+            url: '/img/avatars/pan.png'
+        }, {
+            file: null,
+            data: null,
+            url: '/img/avatars/meat.png'
+        }, {
+            file: null,
+            data: null,
+            url: '/img/avatars/umbrella.png'
+        }, {
+            file: null,
+            data: null,
+            url: '/img/avatars/jar.png'
         }];
         this.avatar = {};
         this.avatarBlob = null;
@@ -82,8 +103,9 @@ app.service('Wizard', [ function () {
 
         this.avatar = avatar;
 
-        // scale image and get the blob now
-        var img = document.getElementById('previewAvatar');
+        // scale image and get the blob now. do not use the previewAvatar element here because it is not updated yet
+        var img = document.createElement('img');
+        img.src = avatar.data || avatar.url;
         var canvas = document.createElement('canvas');
         canvas.width = 256;
         canvas.height = 256;
@@ -122,7 +144,7 @@ app.service('Wizard', [ function () {
     return instance;
 }]);
 
-app.controller('StepController', ['$scope', '$location', 'Wizard', function ($scope, $location, Wizard) {
+app.controller('StepController', ['$scope', '$route', '$location', 'Wizard', function ($scope, $route, $location, Wizard) {
     $scope.wizard = Wizard;
 
     $scope.next = function (page, bad) {
@@ -143,7 +165,7 @@ app.controller('StepController', ['$scope', '$location', 'Wizard', function ($sc
     };
 
     // cheap way to detect if we are in avatar and name selection step
-    if ($('#previewAvatar').get(0) && $('#avatarFileInput').get(0)) {
+    if ($route.current.templateUrl === 'views/setup/step1.html') {
         $('#avatarFileInput').get(0).onchange = function (event) {
             var fr = new FileReader();
             fr.onload = function () {
@@ -161,8 +183,16 @@ app.controller('StepController', ['$scope', '$location', 'Wizard', function ($sc
             fr.readAsDataURL(event.target.files[0]);
         };
 
-        $scope.wizard.setPreviewAvatar($scope.wizard.availableAvatars[0]);
+        // ensure image got loaded before setting the preview avatar
+        var image = document.createElement('img');
+        var randomIndex = Math.floor(Math.random() * $scope.wizard.availableAvatars.length);
+        image.onload = function() {
+            $scope.$apply(function () { $scope.wizard.setPreviewAvatar($scope.wizard.availableAvatars[randomIndex]); });
+            image = null;
+        };
+        image.src = $scope.wizard.availableAvatars[randomIndex].data || $scope.wizard.availableAvatars[randomIndex].url;
     }
+
 }]);
 
 app.controller('FinishController', ['$scope', '$location', '$timeout', 'Wizard', 'Client', function ($scope, $location, $timeout, Wizard, Client) {