Pass along aws config for updates

This fixes crash when auto-updating apps

npm-shrinkwrap.json

@@ -7,6 +7,28 @@
       "from": "https://registry.npmjs.org/async/-/async-1.2.1.tgz",
       "resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz"
     },
+    "aws-sdk": {
+      "version": "2.1.46",
+      "from": "aws-sdk@*",
+      "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1.46.tgz",
+      "dependencies": {
+        "sax": {
+          "version": "0.5.3",
+          "from": "sax@0.5.3",
+          "resolved": "http://registry.npmjs.org/sax/-/sax-0.5.3.tgz"
+        },
+        "xml2js": {
+          "version": "0.2.8",
+          "from": "xml2js@0.2.8",
+          "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz"
+        },
+        "xmlbuilder": {
+          "version": "0.4.2",
+          "from": "xmlbuilder@0.4.2",
+          "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-0.4.2.tgz"
+        }
+      }
+    },
     "body-parser": {
       "version": "1.13.1",
       "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.13.1.tgz",
@@ -105,23 +127,24 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "1.6.0",
+      "version": "1.7.0",
-      "from": "cloudron-manifestformat@1.6.0",
+      "from": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-1.7.0.tgz",
       "dependencies": {
         "java-packagename-regex": {
           "version": "1.0.0",
-          "from": "java-packagename-regex@>=1.0.0 <2.0.0",
+          "from": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz",
           "resolved": "https://registry.npmjs.org/java-packagename-regex/-/java-packagename-regex-1.0.0.tgz"
         },
         "safetydance": {
           "version": "0.0.15",
-          "from": "safetydance@0.0.15",
+          "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz",
           "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.15.tgz"
         },
         "tv4": {
-          "version": "1.1.12",
+          "version": "1.2.3",
-          "from": "tv4@>=1.1.9 <2.0.0",
+          "from": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz",
-          "resolved": "http://registry.npmjs.org/tv4/-/tv4-1.1.12.tgz"
+          "resolved": "https://registry.npmjs.org/tv4/-/tv4-1.2.3.tgz"
         }
       }
     },
@@ -133,15 +156,16 @@
     "connect-lastmile": {
       "version": "0.0.13",
       "from": "connect-lastmile@0.0.13",
+      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
       "dependencies": {
         "debug": {
           "version": "2.1.3",
-          "from": "debug@>=2.1.0 <2.2.0",
+          "from": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
           "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
           "dependencies": {
             "ms": {
               "version": "0.7.0",
-              "from": "ms@0.7.0",
+              "from": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz",
               "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
             }
           }
@@ -2244,7 +2268,7 @@
     },
     "safetydance": {
       "version": "0.0.19",
-      "from": "safetydance@0.0.19",
+      "from": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz",
       "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
     },
     "semver": {

package.json

@@ -17,8 +17,9 @@
   },
   "dependencies": {
     "async": "^1.2.1",
+    "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^1.6.0",
+    "cloudron-manifestformat": "^1.7.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
@@ -68,7 +69,6 @@
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "^2.1.10",
     "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",

setup/INFRA_VERSION

@@ -7,11 +7,11 @@ INFRA_VERSION=8
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.3.1
+BASE_IMAGE=cloudron/base:0.3.3
-MYSQL_IMAGE=cloudron/mysql:0.3.2
+MYSQL_IMAGE=cloudron/mysql:0.3.3
-POSTGRESQL_IMAGE=cloudron/postgresql:0.3.1
+POSTGRESQL_IMAGE=cloudron/postgresql:0.3.2
-MONGODB_IMAGE=cloudron/mongodb:0.3.1
+MONGODB_IMAGE=cloudron/mongodb:0.3.2
-REDIS_IMAGE=cloudron/redis:0.3.1 # if you change this, fix src/addons.js as well
+REDIS_IMAGE=cloudron/redis:0.3.2 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.3.1
+MAIL_IMAGE=cloudron/mail:0.3.2
-GRAPHITE_IMAGE=cloudron/graphite:0.3.3
+GRAPHITE_IMAGE=cloudron/graphite:0.3.4
 

setup/argparser.sh

@@ -16,6 +16,8 @@ arg_tls_key=""
 arg_token=""
 arg_version=""
 arg_web_server_origin=""
+arg_backup_key=""
+arg_aws=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -41,6 +43,12 @@ EOF
         arg_restore_key=$(echo "$2" | $json restoreKey)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
+        arg_backup_key=$(echo "$2" | $json backupKey)
+        [[ "${arg_backup_key}" == "null" ]] && arg_backup_key=""
+
+        arg_aws=$(echo "$2" | $json aws)
+        [[ "${arg_aws}" == "null" ]] && arg_aws=""
+
         shift 2
         ;;
     --) break;;

setup/start.sh

@@ -122,6 +122,7 @@ set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
 echo "Creating cloudron.conf"
+# note that arg_aws is a javascript object and intentionally unquoted below
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
@@ -138,7 +139,9 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "password": "${mysql_root_password}",
         "port": 3306,
         "name": "box"
-    }
+    },
+    "backupKey": "${arg_backup_key}",
+    "aws": ${arg_aws}
 }
 CONF_END
 

src/addons.js

@@ -38,8 +38,7 @@ var appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
     util = require('util'),
     uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
+    vbox = require('./vbox.js');
-    _ = require('underscore');
 
 var NOOP = function (app, callback) { return callback(); };
 
@@ -665,7 +664,7 @@ function setupRedis(app, callback) {
         name: 'redis-' + app.id,
         Hostname: config.appFqdn(app.location),
         Tty: true,
-        Image: 'cloudron/redis:0.3.1',
+        Image: 'cloudron/redis:0.3.2', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
         Volumes: {},
         VolumesFrom: []

src/appdb.js

@@ -35,12 +35,13 @@ exports = module.exports = {
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
-    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
     RSTATE_STOPPED: 'stopped', // app stopped by use
+    RSTATE_ERROR: 'error',
 
+    // run codes (keep in sync in UI)
     HEALTH_HEALTHY: 'healthy',
     HEALTH_UNHEALTHY: 'unhealthy',
     HEALTH_ERROR: 'error',
@@ -335,6 +336,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
 
     // Rules are:
     // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
     // update and configure are allowed only in installed state
 

src/apps.js

@@ -490,10 +490,11 @@ function restore(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var restoreConfig = app.lastBackupConfig;
+        // restore without a backup is the same as re-install
-        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+        var restoreConfig = app.lastBackupConfig, values = { };
-
+        if (restoreConfig) {
-        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
             error = checkManifestConstraints(restoreConfig.manifest);
             if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
@@ -501,7 +502,7 @@ function restore(appId, callback) {
             if (error) return callback(error);
 
             // ## should probably query new location, access restriction from user
-        var values = {
+            values = {
                 manifest: restoreConfig.manifest,
                 portBindings: restoreConfig.portBindings,
 
@@ -512,6 +513,7 @@ function restore(appId, callback) {
                     manifest: app.manifest
                 }
             };
+        }
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -573,6 +575,8 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
         return new Error('Box version exceeds Apps maxBoxVersion');
     }
@@ -664,7 +668,7 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
                 return iteratorDone();
             }
 
-           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
                 if (error) debug('Error initiating autoupdate of %s', appId);
 
                 iteratorDone(null);
@@ -700,7 +704,7 @@ function backupApp(app, addonsToBackup, callback) {
         return callback(safe.error);
     }
 
-    backups.getBackupUrl(app, null, function (error, result) {
+    backups.getBackupUrl(app, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -709,7 +713,7 @@ function backupApp(app, addonsToBackup, callback) {
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
             addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -756,7 +760,7 @@ function restoreApp(app, addonsToRestore, callback) {
 
         debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
 
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             addons.restoreAddons(app, addonsToRestore, callback);

src/apptask.js

@@ -248,7 +248,7 @@ function deleteImage(app, manifest, callback) {
             noprune: false
         };
 
-        // delete image by id because docker pull pulls down all the tags and this is the only way to delete all tags
+         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
         docker.getImage(result.Id).remove(removeOptions, function (error) {
             if (error && error.statusCode === 404) return callback(null);
             if (error && error.statusCode === 409) return callback(null); // another container using the image
@@ -331,8 +331,12 @@ function startContainer(app, callback) {
             vbox.forwardFromHostToVirtualBox(app.id + '-tcp' + containerPort, hostPort);
         }
 
+        var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+
         var startOptions = {
             Binds: addons.getBindsSync(app, app.manifest.addons),
+            Memory: memoryLimit / 2,
+            MemorySwap: memoryLimit, // Memory + Swap
             PortBindings: dockerPortBindings,
             PublishAllPorts: false,
             Links: addons.getLinksSync(app, app.manifest.addons),
@@ -355,6 +359,11 @@ function startContainer(app, callback) {
 }
 
 function stopContainer(app, callback) {
+    if (!app.containerId) {
+        debugApp(app, 'No previous container to stop');
+        return callback();
+    }
+
     var container = docker.getContainer(app.containerId);
     debugApp(app, 'Stopping container %s', container.id);
 
@@ -532,7 +541,7 @@ function install(app, callback) {
         deleteVolume.bind(null, app),
         unregisterSubdomain.bind(null, app),
         removeOAuthProxyCredentials.bind(null, app),
-        removeIcon.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
         unconfigureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
@@ -617,7 +626,11 @@ function restore(app, callback) {
          // oldConfig can be null during upgrades
         addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
         deleteVolume.bind(null, app),
-        deleteImage.bind(null, app, app.manifest),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
         unconfigureNginx.bind(null, app),
@@ -671,6 +684,7 @@ function restore(app, callback) {
 
 // note that configure is called after an infra update as well
 function configure(app, callback) {
+    // oldConfig can be null during an infra update
     var locationChanged = app.oldConfig ? app.oldConfig.location !== app.location : true;
 
     async.series([
@@ -753,7 +767,11 @@ function update(app, callback) {
         stopApp.bind(null, app),
         deleteContainer.bind(null, app),
         addons.teardownAddons.bind(null, app, unusedAddons),
-        deleteImage.bind(null, app, app.manifest), // delete image even if did not change (see df158b111f)
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             deleteImage(app, app.oldConfig.manifest, done);
+        },
         // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
 
         function (next) {

src/aws.js

@@ -0,0 +1,122 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    AWSError: AWSError,
+
+    getAWSCredentials: getAWSCredentials,
+
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('./config.js'),
+    debug = require('debug')('box:aws'),
+    superagent = require('superagent'),
+    util = require('util');
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function AWSError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AWSError, Error);
+AWSError.INTERNAL_ERROR = 'Internal Error';
+AWSError.MISSING_CREDENTIALS = 'Missing AWS credentials';
+
+function getAWSCredentials(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getAWSCredentials()');
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
+        superagent.get(url).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 201) return callback(new Error(result.text));
+            if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
+
+            debug('getAWSCredentials()', result.body.credentials);
+
+            return callback(null, {
+                accessKeyId: result.body.credentials.AccessKeyId,
+                secretAccessKey: result.body.credentials.SecretAccessKey,
+                sessionToken: result.body.credentials.SessionToken,
+                region: 'us-east-1'
+            });
+        });
+    } else {
+        if (!config.aws().accessKeyId || !config.aws().secretAccessKey) return callback(new AWSError(AWSError.MISSING_CREDENTIALS));
+
+        callback(null, {
+            accessKeyId: config.aws().accessKeyId,
+            secretAccessKey: config.aws().secretAccessKey,
+            region: 'us-east-1'
+        });
+    }
+}
+
+function getSignedUploadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedUploadUrl()');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedDownloadUrl()');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}

src/backups.js

@@ -10,6 +10,7 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    aws = require('./aws.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
     superagent = require('superagent'),
@@ -54,42 +55,50 @@ function getAllPaged(page, perPage, callback) {
     });
 }
 
-function getBackupUrl(app, appBackupIds, callback) {
+function getBackupUrl(app, callback) {
     assert(!app || typeof app === 'object');
-    assert(!appBackupIds || util.isArray(appBackupIds));
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+    var filename = '';
+    if (app) {
+        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    } else {
+        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
+    }
 
-    var data = {
+    aws.getSignedUploadUrl(filename, function (error, result) {
-        boxVersion: config.version(),
+        if (error) return callback(error);
-        appId: app ? app.id : null,
+
-        appVersion: app ? app.manifest.version : null,
+        var obj = {
-        appBackupIds: appBackupIds
+            id: filename,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
         };
 
-    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
+        debug('getBackupUrl: ', obj);
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
 
-        return callback(null, result.body);
+        callback(null, obj);
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+    aws.getSignedDownloadUrl(backupId, function (error, result) {
+        if (error) return callback(error);
 
-    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
+        var obj = {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+            id: backupId,
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+            url: result.url,
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
 
-        return callback(null, result.body);
+        debug('getRestoreUrl: ', obj);
+
+        callback(null, obj);
     });
 }
-
-

src/cloudron.js

@@ -25,6 +25,7 @@ var apps = require('./apps.js'),
     AppsError = require('./apps.js').AppsError,
     assert = require('assert'),
     async = require('async'),
+    aws = require('./aws.js'),
     backups = require('./backups.js'),
     BackupsError = require('./backups.js').BackupsError,
     clientdb = require('./clientdb.js'),
@@ -46,7 +47,8 @@ var apps = require('./apps.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
 
 var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
@@ -271,8 +273,7 @@ function getConfig(callback) {
 function sendHeartbeat() {
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
 
-    // TODO: this must be a POST
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
-    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
         if (error) debug('Error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
         else debug('Heartbeat sent to %s', url);
@@ -492,19 +493,20 @@ function doUpdate(boxUpdateInfo, callback) {
             var args = {
                 sourceTarballUrl: result.body.url,
 
-                // IMPORTANT: if you change this, fix up argparser.sh as well. keep these sorted for readability
+                // this data is opaque to the installer
                 data: {
-                    apiServerOrigin: config.apiServerOrigin(),
                     boxVersionsUrl: config.get('boxVersionsUrl'),
-                    fqdn: config.fqdn(),
-                    isCustomDomain: config.isCustomDomain(),
-                    restoreKey: null,
-                    restoreUrl: null,
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
-                    token: config.token(),
                     version: boxUpdateInfo.version,
-                    webServerOrigin: config.webServerOrigin()
+                    apiServerOrigin: config.apiServerOrigin(),
+                    webServerOrigin: config.webServerOrigin(),
+                    fqdn: config.fqdn(),
+                    token: config.token(),
+                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    isCustomDomain: config.isCustomDomain(),
+                    restoreUrl: null,
+                    restoreKey: null,
+                    aws: config.aws()
                 }
             };
 
@@ -561,7 +563,7 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+    backups.getBackupUrl(null /* app */, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -569,16 +571,19 @@ function backupBoxWithAppBackupIds(appBackupIds, callback) {
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             debug('backup: successful');
 
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
                 callback(null, result.id);
             });
         });
+    });
 }
 
 // this function expects you to have a lock
@@ -609,7 +614,7 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, 'Backing up app at ' + app.location);
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
 
                 if (error && error.reason === AppsError.BAD_STATE) {
                     debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);

src/config.js

@@ -30,6 +30,9 @@ exports = module.exports = {
 
     isDev: isDev,
 
+    backupKey: backupKey,
+    aws: aws,
+
     // for testing resets to defaults
     _reset: initConfig
 };
@@ -70,6 +73,13 @@ function initConfig() {
     data.webServerOrigin = null;
     data.internalPort = 3001;
     data.ldapPort = 3002;
+    data.backupKey = 'backupKey';
+    data.aws = {
+        backupBucket: null,
+        backupPrefix: null,
+        accessKeyId: null,      // selfhosting only
+        secretAccessKey: null   // selfhosting only
+    };
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -172,3 +182,10 @@ function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
 
+function backupKey() {
+    return get('backupKey');
+}
+
+function aws() {
+    return get('aws');
+}

src/ldap.js

@@ -25,7 +25,7 @@ var gLogger = {
 };
 
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
-var GROUP_ADMINS_DN = 'cn=admin,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
 
 function start(callback) {
     assert(typeof callback === 'function');

src/routes/oauth2.js

@@ -171,6 +171,8 @@ function sendErrorPageOrRedirect(req, res, message) {
     }
 }
 
+// use this instead of sendErrorPageOrRedirect(), in case we have a returnTo provided in the query, to avoid login loops
+// This usually happens when the OAuth client ID is wrong
 function sendError(req, res, message) {
     assert.strictEqual(typeof req, 'object');
     assert.strictEqual(typeof res, 'object');

src/routes/test/backups-test.js

@@ -119,8 +119,8 @@ describe('Backups API', function () {
 
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
-                        .put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] })
+                        .get('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
-                        .reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                        .reply(201, { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', sessionToken: 'sessionToken' });
 
             request.post(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })

src/routes/test/cloudron-test.js

@@ -450,8 +450,18 @@ describe('Cloudron', function () {
         });
 
         it('fails when in wrong state', function (done) {
-            var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { size: 'small', region: 'sfo', restoreKey: 'someid' }).reply(409, {});
+            var scope2 = nock(config.apiServerOrigin())
-            var scope2 = nock(config.apiServerOrigin()).put('/api/v1/boxes/' + config.fqdn() + '/backupurl?token=APPSTORE_TOKEN', { boxVersion: '0.5.0', appId: null, appVersion: null, appBackupIds: [] }).reply(201, { id: 'someid', url: 'http://foobar', backupKey: 'somerestorekey' });
+                    .get('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .reply(201, { credentials: { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', sessionToken: 'sessionToken' } });
+
+            var scope3 = nock(config.apiServerOrigin())
+                    .filteringRequestBody(function () { return false; })
+                    .post('/api/v1/boxes/' + config.fqdn() + '/backupDone?token=APPSTORE_TOKEN')
+                    .reply(200, { id: 'someid' });
+
+            var scope1 = nock(config.apiServerOrigin())
+                .filteringRequestBody(function () { return false; })
+                .post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', { }).reply(409, {});
 
             injectShellMock();
 
@@ -463,7 +473,7 @@ describe('Cloudron', function () {
                 expect(result.statusCode).to.equal(202);
 
                 function checkAppstoreServerCalled() {
-                    if (scope1.isDone() && scope2.isDone()) {
+                    if (scope1.isDone() && scope2.isDone() && scope3.isDone()) {
                         restoreShellMock();
                         return done();
                     }

src/scripts/backupapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: backup.sh <appid> <url> <key>"
+    echo "Usage: backupapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -22,6 +22,7 @@ readonly DATA_DIR="${HOME}/data"
 app_id="$1"
 backup_url="$2"
 backup_key="$3"
+session_token="$4"
 readonly now=$(date "+%Y-%m-%dT%H:%M:%S")
 readonly app_data_dir="${DATA_DIR}/${app_id}"
 readonly app_data_snapshot="${DATA_DIR}/snapshots/${app_id}-${now}"
@@ -31,9 +32,17 @@ btrfs subvolume snapshot -r "${app_data_dir}" "${app_data_snapshot}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${app_data_snapshot}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT "${headers[@]}" --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/backupbox.sh

@@ -13,12 +13,13 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 2 ]; then
-    echo "Usage: backupbox.sh <url> <key>"
+    echo "Usage: backupbox.sh <url> <key> [aws session token]"
     exit 1
 fi
 
 backup_url="$1"
 backup_key="$2"
+session_token="$3"
 now=$(date "+%Y-%m-%dT%H:%M:%S")
 BOX_DATA_DIR="${HOME}/data/box"
 box_snapshot_dir="${HOME}/data/snapshots/box-${now}"
@@ -32,9 +33,17 @@ btrfs subvolume snapshot -r "${BOX_DATA_DIR}" "${box_snapshot_dir}"
 for try in `seq 1 5`; do
     echo "Uploading backup to ${backup_url} (try ${try})"
     error_log=$(mktemp)
+
+    headers=("-H" "Content-Type:")
+
+    # federated tokens in CaaS case need session token
+    if [ ! -z "$session_token" ]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
     if tar -cvzf - -C "${box_snapshot_dir}" . \
            | openssl aes-256-cbc -e -pass "pass:${backup_key}" \
-           | curl --fail -H "Content-Type:" -X PUT --data-binary @- "${backup_url}" 2>"${error_log}"; then
+           | curl --fail -X PUT ${headers[@]} --data-binary @- "${backup_url}" 2>"${error_log}"; then
         break
     fi
     cat "${error_log}" && rm "${error_log}"

src/scripts/restoreapp.sh

@@ -13,7 +13,7 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [ $# -lt 3 ]; then
-    echo "Usage: restoreapp.sh <appid> <url> <key>"
+    echo "Usage: restoreapp.sh <appid> <url> <key> [aws session token]"
     exit 1
 fi
 
@@ -23,6 +23,7 @@ readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max
 app_id="$1"
 restore_url="$2"
 restore_key="$3"
+session_token="$4"
 
 echo "Downloading backup: ${restore_url} and key: ${restore_key}"
 
@@ -30,7 +31,14 @@ for try in `seq 1 5`; do
     echo "Download backup from ${restore_url} (try ${try})"
     error_log=$(mktemp)
 
-    if $curl -L "${restore_url}" \
+    headers=("") # empty element required (http://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u)
+
+    # federated tokens in CaaS case need session token
+    if [[ ! -z "${session_token}" ]]; then
+        headers=(${headers[@]} "-H" "x-amz-security-token: ${session_token}")
+    fi
+
+    if $curl -L "${headers[@]}" "${restore_url}" \
         | openssl aes-256-cbc -d -pass "pass:${restore_key}" \
         | tar -zxf - -C "${DATA_DIR}/${app_id}" 2>"${error_log}"; then
         chown -R yellowtent:yellowtent "${DATA_DIR}/${app_id}"

src/webhooks.js

@@ -0,0 +1,47 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    backupDone: backupDone
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:webhooks'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function backupDone(filename, app, appBackupIds, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert(!app || typeof app === 'object');
+    assert(!appBackupIds || util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('backupDone():', filename);
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupDone';
+        var data = {
+            boxVersion: config.version(),
+            restoreKey: filename,
+            appId: app ? app.id : null,
+            appVersion: app ? app.manifest.version : null,
+            appBackupIds: appBackupIds
+        };
+
+        superagent.post(url).send(data).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 200) return callback(new Error(result.text));
+            if (!result.body) return callback(new Error('Unexpected response'));
+
+            debug('backupDone()', filename);
+
+            return callback(null);
+        });
+    } else {
+        // TODO call custom webhook
+        callback(null);
+    }
+}

webadmin/src/appstatus.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron App Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/error.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron Error </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/index.html

@@ -8,6 +8,9 @@
 
     <link href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -48,9 +51,6 @@
     <!-- Main Application -->
     <script src="js/index.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body>
@@ -81,7 +81,7 @@
                         <li ng-repeat="change in config.update.box.changelog">{{change}}</li>
                     </ul>
                     <br/>
-                    <fieldset>
+                    <fieldset ng-show="installedApps | readyToUpdate">
                         <form name="update_form" role="form" ng-submit="doUpdate()" autocomplete="off">
                             <div class="form-group" ng-class="{ 'has-error': update_form.password.$dirty && update_form.password.$invalid }">
                                 <label class="control-label" for="inputUpdatePassword">Give your password to verify that you are performing that action</label>
@@ -99,7 +99,7 @@
                 </div>
                 <div class="modal-footer">
                     <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
+                    <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="update_form.$invalid || update.busy" ng-show="installedApps | readyToUpdate"><i class="fa fa-spinner fa-pulse" ng-show="update.busy"></i> Update</button>
                 </div>
             </div>
         </div>

webadmin/src/nakeddomain.html

@@ -6,13 +6,13 @@
 
     <title> Cloudron </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet" type="text/css">
+
     <!-- external fonts and CSS -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet" type="text/css">
-
     <!-- jQuery-->
     <script src="3rdparty/js/jquery.min.js"></script>
 

webadmin/src/setup.html

@@ -6,6 +6,9 @@
 
     <title> Cloudron Setup </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -31,9 +34,6 @@
     <!-- Setup Application -->
     <script src="js/setup.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body class="setup" ng-app="Application" ng-controller="SetupController">

webadmin/src/theme.scss

@@ -195,8 +195,8 @@ html {
 
 .appstore-item-badge-testing {
     position: absolute;
-    right: 15px;
+    right: 0;
-    top: 15px;
+    top: 2px;
 }
 
 .appstore-item-content-testing {
@@ -330,6 +330,10 @@ html {
     background-color: #5CB85C;
 }
 
+.badge-warning {
+    background-color: #EFBD48;
+}
+
 .badge-danger {
     background-color: $brand-danger;
 }

webadmin/src/update.html

@@ -6,6 +6,9 @@
 
     <title> Cloudron Update </title>
 
+    <!-- Theme CSS -->
+    <link href="theme.css" rel="stylesheet">
+
     <!-- Custom Fonts -->
     <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
     <link href="//fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
@@ -23,9 +26,6 @@
     <!-- Update Application -->
     <script src="js/update.js"></script>
 
-    <!-- Theme CSS -->
-    <link href="theme.css" rel="stylesheet">
-
 </head>
 
 <body ng-app="Application" ng-controller="Controller" style="background-color: #7F7F7F">

webadmin/src/views/apps.html

@@ -269,3 +269,6 @@
         </div>
     </div>
 </div>
+
+<!-- Offset the footer -->
+<br/><br/>

webadmin/src/views/appstore.html

@@ -147,8 +147,9 @@
         <div class="col-md-10" ng-show="ready && apps.length">
             <div class="row-no-margin">
                 <div class="col-sm-1 appstore-item" ng-repeat="app in apps">
-                    <div class="appstore-item-content highlight" ng-click="showInstall(app)" ng-class="{ 'appstore-item-content-testing': app.publishState === 'testing' }">
+                    <div class="appstore-item-content highlight" ng-click="showInstall(app)" ng-class="{ 'appstore-item-content-testing': (app.publishState === 'testing' || app.publishState === 'pending_approval') }">
                         <span class="badge badge-danger appstore-item-badge-testing" ng-show="app.publishState === 'testing'">Testing</span>
+                        <span class="badge badge-warning appstore-item-badge-testing" ng-show="app.publishState === 'pending_approval'">Pending Approval</span>
                         <div class="appstore-item-content-icon col-same-height">
                             <img ng-src="{{app.iconUrl}}" onerror="this.onerror=null;this.src='img/appicon_fallback.png'" class="app-icon"/>
                         </div>

webadmin/src/views/graphs.js

@@ -98,7 +98,7 @@ angular.module('Application').controller('GraphsController', ['$scope', '$locati
             var options = {
                 scaleOverride: true,
                 scaleSteps: 10,
-                scaleStepWidth: $scope.activeApp === 'system' ? 100 : 10,
+                scaleStepWidth: $scope.activeApp === 'system' ? 200 : 20,
                 scaleStartValue: 0
             };