convert cert to pem

this is not used anywhere since we want to wait for rate limits to be
fixed.

The current limits are :

    Rate limit on registrations per IP is currently 10 per 3 hours
    Rate limit on certificates per Domain is currently 5 per 7 days

The domains are counted based on https://publicsuffix.org/list/ (not TLD). Like appspot.com, herokuapp.com while not a TLD, it a public suffix. This list allows browser authors to limit how cookies can be manipulated by the subdomain of those domains. like app1.appspot.com cannot go and change things of app2.appspot.com.

This means
a) we cannot use LE for cloudron.me, cloudron.us (or we have to get on that list)

b) even for custom domains we get only 5 certs every 7 days. And one of them is taken for my.xx domain.

https://community.letsencrypt.org/t/public-beta-rate-limits/4772/38

crashnotifier.js

@@ -36,12 +36,7 @@ function main() {
     var processName = process.argv[2];
     console.log('Started crash notifier for', processName);
 
-    mailer.initialize(function (error) {
-        if (error) return console.error(error);
-
-        sendCrashNotification(processName);
-    });
+    sendCrashNotification(processName);
 }
 
 main();
-

migrations/20151113091257-apps-alter-manifestJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151126111337-apps-alter-jsons.js

@@ -0,0 +1,19 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson TEXT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson VARCHAR(2048)')
+    ], callback);
+};

migrations/schema.sql

@@ -45,18 +45,18 @@ CREATE TABLE IF NOT EXISTS apps(
     runState VARCHAR(512),
     health VARCHAR(128),
     containerId VARCHAR(128),
-    manifestJson VARCHAR(2048),
+    manifestJson TEXT,
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestrictionJson VARCHAR(2048),
+    accessRestrictionJson TEXT,
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
     lastBackupId VARCHAR(128),
-    lastBackupConfigJson VARCHAR(2048), // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
 
-    oldConfigJson VARCHAR(2048), // used to pass old config for apptask
+    oldConfigJson TEXT, // used to pass old config for apptask
 
     PRIMARY KEY(id));
 

package.json

@@ -10,7 +10,7 @@
     "type": "git"
   },
   "engines": [
-    "node >= 0.12.0"
+    "node >=4.0.0 <=4.1.1"
   ],
   "dependencies": {
     "async": "^1.2.1",
@@ -58,7 +58,7 @@
     "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
     "superagent": "~0.21.0",
-    "supererror": "^0.7.0",
+    "supererror": "^0.7.1",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
     "underscore": "^1.7.0",
     "valid-url": "^1.0.9",

setup/INFRA_VERSION

@@ -3,17 +3,17 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=20
+INFRA_VERSION=21
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.7.0
-MYSQL_IMAGE=cloudron/mysql:0.7.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.7.0
-MONGODB_IMAGE=cloudron/mongodb:0.7.0
-REDIS_IMAGE=cloudron/redis:0.7.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.8.0
-GRAPHITE_IMAGE=cloudron/graphite:0.7.0
+BASE_IMAGE=cloudron/base:0.8.0
+MYSQL_IMAGE=cloudron/mysql:0.8.0
+POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
+MONGODB_IMAGE=cloudron/mongodb:0.8.0
+REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
+MAIL_IMAGE=cloudron/mail:0.9.0
+GRAPHITE_IMAGE=cloudron/graphite:0.8.0
 
 MYSQL_REPO=cloudron/mysql
 POSTGRESQL_REPO=cloudron/postgresql

setup/argparser.sh

@@ -16,8 +16,7 @@ arg_tls_key=""
 arg_token=""
 arg_version=""
 arg_web_server_origin=""
-arg_backup_key=""
-arg_aws=""
+arg_backup_config=""
 arg_dns_config=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
@@ -38,17 +37,14 @@ EOF
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
 
-        arg_restore_url=$(echo "$2" | $json restoreUrl)
+        arg_restore_url=$(echo "$2" | $json restore.url)
         [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
 
-        arg_restore_key=$(echo "$2" | $json restoreKey)
+        arg_restore_key=$(echo "$2" | $json restore.key)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
-        arg_backup_key=$(echo "$2" | $json backupKey)
-        [[ "${arg_backup_key}" == "null" ]] && arg_backup_key=""
-
-        arg_aws=$(echo "$2" | $json aws)
-        [[ "${arg_aws}" == "null" ]] && arg_aws=""
+        arg_backup_config=$(echo "$2" | $json backupConfig)
+        [[ "${arg_backup_config}" == "null" ]] && arg_backup_config=""
 
         arg_dns_config=$(echo "$2" | $json dnsConfig)
         [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""

setup/start.sh

@@ -48,6 +48,7 @@ mkdir -p "${DATA_DIR}/mongodb"
 mkdir -p "${DATA_DIR}/snapshots"
 mkdir -p "${DATA_DIR}/addons"
 mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${DATA_DIR}/acme"
 
 # bookkeep the version as part of data
 echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
@@ -106,15 +107,26 @@ ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs
     -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
 
 # generate these for update code paths as well to overwrite splash
+admin_cert_file="cert/host.cert"
+admin_key_file="cert/host.key"
+if [[ -f "${DATA_DIR}/box/certs/admin.cert" && -f "${DATA_DIR}/box/certs/admin.key" ]]; then
+    admin_cert_file="${DATA_DIR}/box/certs/admin.cert"
+    admin_key_file="${DATA_DIR}/box/certs/admin.key"
+fi
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
 mkdir -p "${DATA_DIR}/nginx/cert"
-echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
-echo "${arg_tls_key}" > ${DATA_DIR}/nginx/cert/host.key
+if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
+    cp "${DATA_DIR}/box/certs/host.cert" "${DATA_DIR}/nginx/cert/host.cert"
+    cp "${DATA_DIR}/box/certs/host.key" "${DATA_DIR}/nginx/cert/host.key"
+else
+    echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
+    echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
+fi
 
 set_progress "33" "Changing ownership"
-chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
 chown "${USER}:${USER}" "${DATA_DIR}"
 
 set_progress "40" "Setting up infra"
@@ -124,7 +136,6 @@ set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
 set -eu
 echo "Creating cloudron.conf"
-# note that arg_aws is a javascript object and intentionally unquoted below
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
@@ -141,9 +152,7 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
         "password": "${mysql_root_password}",
         "port": 3306,
         "name": "box"
-    },
-    "backupKey": "${arg_backup_key}",
-    "aws": ${arg_aws}
+    }
 }
 CONF_END
 
@@ -155,6 +164,14 @@ cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
 CONF_END
 EOF
 
+# Add Backup Configuration
+if [[ ! -z "${arg_backup_config}" ]]; then
+    echo "Add Backup Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
+fi
+
 # Add DNS Configuration
 if [[ ! -z "${arg_dns_config}" ]]; then
     echo "Add DNS Config"

setup/start/nginx/appconfig.ejs

@@ -37,7 +37,8 @@ server {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
 
-    error_page 500 502 503 504 @appstatus;
+    # only serve up the status page if we get proxy gateway errors
+    error_page 502 503 504 @appstatus;
     location @appstatus {
         return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
     }

setup/start/nginx/nginx.ejs

@@ -38,6 +38,11 @@ http {
             deny all;
         }
 
+        # acme challenges
+        location /.well-known/acme-challenge/ {
+            alias /home/yellowtent/data/acme/;
+        }
+
         location / {
             # redirect everything to HTTPS
             return 301 https://$host$request_uri;

src/acme.js

@@ -0,0 +1,334 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('acme'),
+    execSync = require('child_process').execSync,
+    fs = require('fs'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    superagent = require('superagent'),
+    urlBase64Encode = require('url-base64-node').escape,
+    ursa = require('ursa'),
+    util = require('util'),
+    _ = require('underscore');
+
+var CA_STAGING = 'https://acme-v01.api.letsencrypt.org',
+    CA_STAGING = 'https://acme-staging.api.letsencrypt.org/',
+    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+function AcmeError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AcmeError, Error);
+AcmeError.INTERNAL_ERROR = 'Internal Error';
+AcmeError.EXTERNAL_ERROR = 'External Error';
+AcmeError.ALREADY_EXISTS = 'Already Exists';
+AcmeError.NOT_COMPLETED = 'Not Completed';
+AcmeError.FORBIDDEN = 'Forbidden';
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function getNonce(callback) {
+    superagent.get(CA_STAGING + '/directory', function (error, response) {
+        if (error) return callback(error);
+        if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
+
+        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
+    });
+}
+
+// urlsafe base64 encoding (jose)
+function b64(str) {
+    var buf = util.isBuffer(str) ? str : new Buffer(str);
+   return urlBase64Encode(buf.toString('base64'));
+}
+
+function sendSignedRequest(url, privateKeyPem, payload, callback) {
+    assert.strictEqual(typeof url, 'string');
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof payload, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var privateKey = ursa.createPrivateKey(privateKeyPem);
+
+    var header = {
+        alg: 'RS256',
+        jwk: {
+            e: b64(privateKey.getExponent()),
+            kty: 'RSA',
+            n: b64(privateKey.getModulus())
+        }
+    };
+ 
+    var payload64 = b64(payload);
+
+    getNonce(function (error, nonce) {
+        if (error) return callback(error);
+
+        debug('Using nonce %s', nonce);
+
+        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+        var signer = ursa.createSigner('sha256');
+        signer.update(protected64 + '.' + payload64, 'utf8');
+        var signature64 = urlBase64Encode(signer.sign(privateKey, 'base64'));
+
+        var data = {
+            header: header,
+            protected: protected64,
+            payload: payload64,
+            signature: signature64
+        };
+
+        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).buffer().end(function (error, res) {
+            if (error && !error.response) return callback(error); // network errors
+
+            callback(null, res);
+        });
+    });
+}
+
+function registerUser(privateKeyPem, email, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-reg',
+        contact: [ 'mailto:' + email ],
+        agreement: LE_AGREEMENT
+    };
+
+    debug('registerUser: %s', email);
+
+    sendSignedRequest(CA_STAGING + '/acme/new-reg', privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
+        if (result.statusCode === 409) return callback(new AcmeError(AcmeError.ALREADY_EXISTS, result.body.detail));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerUser: registered user %s', email);
+
+        callback();
+    });
+}
+
+function registerDomain(privateKeyPem, domain, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-authz',
+        identifier: {
+            type: 'dns',
+            value: domain
+        }
+    };
+
+    debug('registerDomain: %s', domain);
+
+    sendSignedRequest(CA_STAGING + '/acme/new-authz', privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering domain: ' + error.message));
+        if (result.statusCode === 403) return callback(new AcmeError(AcmeError.FORBIDDEN, result.body.detail));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerDomain: registered %s', domain);
+
+        callback(null, result.body);
+    });
+}
+
+function prepareHttpChallenge(privateKeyPem, challenge, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    var token = challenge.token;
+
+    var privateKey = ursa.createPrivateKey(privateKeyPem);
+
+    var jwk = {
+        e: b64(privateKey.getExponent()),
+        kty: 'RSA',
+        n: b64(privateKey.getModulus())
+    };
+
+    var shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    var thumbprint = urlBase64Encode(shasum.digest('base64'));
+    var keyAuthorization = token + '.' + thumbprint;
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, token));
+
+    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, token), token + '.' + thumbprint, function (error) {
+        if (error) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function notifyChallengeReady(privateKeyPem, challenge, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('notifyChallengeReady: %s was met', challenge.uri);
+
+    var keyAuthorization = fs.readFileSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), 'utf8');
+
+    var payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    sendSignedRequest(challenge.uri, privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when notifying challenge: ' + error.message));
+        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 202, got %s %s', result.statusCode, result.text)));
+
+        callback();
+    });
+}
+
+function waitForChallenge(challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('waitingForChallenge: %j', challenge);
+
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        debug('waitingForChallenge: getting status');
+
+        superagent.get(challenge.uri).end(function (error, result) {
+            if (error && !error.response) {
+                debug('waitForChallenge: network error getting uri %s', challenge.uri);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, error.message)); // network error
+            }
+            if (result.statusCode !== 202) {
+                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+            }
+
+            debug('waitForChallenge: status is "%s"', result.body.status);
+
+            if (result.body.status === 'pending') return retryCallback(new AcmeError(AcmeError.NOT_COMPLETED));
+            else if (result.body.status === 'valid') return retryCallback();
+            else return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
+        });
+    }, callback);
+}
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+function signCertificate(privateKeyPem, certificateDer, callback) {
+    assert(util.isBuffer(privateKeyPem));
+    assert(util.isBuffer(certificateDer));
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-cert',
+        csr: b64(certificateDer)
+    };
+
+    debug('signCertificate: signing %s', payload.csr);
+
+    sendSignedRequest(CA_STAGING + '/acme/new-cert', privateKeyPem, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when signing certificate: ' + error.message));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        // TODO: result.body can be empty in which case it has to be polled for from this location
+        debug('signCertificate: certificate is available at ', result.headers['location']);
+
+        callback(null, result.text);
+    });
+}
+
+function acmeFlow(domain, email, privateKeyPem, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof email, 'string');
+    assert(util.isBuffer(privateKeyPem));
+    assert.strictEqual(typeof callback, 'function');
+
+    registerUser(privateKeyPem, email, function (error) {
+        if (error && error.reason !== AcmeError.ALREADY_EXISTS) return callback(error);
+
+        registerDomain(privateKeyPem, domain, function (error, result) {
+            if (error) return callback(error);
+
+            debug('getCertificate: challenges: %j', result);
+
+            var httpChallenges = result.challenges.filter(function(x) { return x.type === 'http-01'; });
+            if (httpChallenges.length === 0) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'no http challenges'));
+            var challenge = httpChallenges[0];
+
+            prepareHttpChallenge(privateKeyPem, challenge, function (error) {
+                if (error) return callback(error);
+
+                notifyChallengeReady(privateKeyPem, challenge, function (error) {
+                    if (error) return callback(error);
+
+                    waitForChallenge(challenge, function (error) {
+                        if (error) return callback(error);
+
+                        var serverKey = execSync('openssl genrsa 4096');
+                        var certificateDer = execSync(util.format('openssl req -nodes -outform DER -subj /CN=%s', domain), { stdio: [ serverKey, null, null ] });
+
+                        signCertificate(privateKeyPem, certificateDer, function (error, certificateDer) {
+                            if (error) return callback(error);
+                            var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { stdio: [ certificateDer, null, null ] });
+
+                            callback(null, serverKey, certificatePem);
+                        });
+                    });
+                });
+            });
+        });
+    });
+}
+
+function getCertificate(domain, callback) {
+    var email = 'admin@' + config.fqdn();
+    var privateKeyPem;
+
+    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
+        debug('getCertificate: generating acme account key on first run');
+        privateKeyPem = execSync('openssl genrsa 4096');
+        fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, privateKeyPem);
+    } else {
+        privateKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
+    }
+
+    acmeFlow(domain, email, privateKeyPem, callback);
+}
+
+getCertificate('foobar.girish.in', function (error, key, cert) {
+    console.dir(error);
+    console.dir(key);
+    console.dir(cert);
+});

src/addons.js

@@ -325,10 +325,10 @@ function setupSimpleAuth(app, options, callback) {
             if (error) return callback(error);
 
             var env = [
-                'SIMPLE_AUTH_SERVER=172.17.42.1',
+                'SIMPLE_AUTH_SERVER=172.17.0.1',
                 'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_URL=http://172.17.42.1:' + config.get('simpleAuthPort'), // obsolete, remove
-                'SIMPLE_AUTH_ORIGIN=http://172.17.42.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
                 'SIMPLE_AUTH_CLIENT_ID=' + id
             ];
 
@@ -359,9 +359,9 @@ function setupLdap(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'LDAP_SERVER=172.17.42.1',
+        'LDAP_SERVER=172.17.0.1',
         'LDAP_PORT=' + config.get('ldapPort'),
-        'LDAP_URL=ldap://172.17.42.1:' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
         'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
         'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
@@ -391,7 +391,7 @@ function setupSendMail(app, options, callback) {
     var env = [
         'MAIL_SMTP_SERVER=mail',
         'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + (app.location || app.id), // use app.id for bare domains
+        'MAIL_SMTP_USERNAME=' + (app.location || app.id) + '-app', // use app.id for bare domains
         'MAIL_DOMAIN=' + config.fqdn()
     ];
 
@@ -778,7 +778,7 @@ function setupRedis(app, options, callback) {
         name: 'redis-' + app.id,
         Hostname: 'redis-' + app.location,
         Tty: true,
-        Image: 'cloudron/redis:0.7.0', // if you change this, fix setup/INFRA_VERSION as well
+        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
         Volumes: {
             '/tmp': {},

src/apphealthmonitor.js

@@ -110,7 +110,11 @@ function processApps(callback) {
         async.each(apps, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            debug('apps alive: [%s]', apps.map(function (a) { return a.location; }).join(', '));
+            var alive = apps
+                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
+                .map(function (a) { return a.location; }).join(', ');
+
+            debug('apps alive: [%s]', alive);
 
             callback(null);
         });
@@ -151,7 +155,7 @@ function processDockerEvents() {
                 debug('OOM Context: %s', context);
 
                 // do not send mails for dev apps
-                if (app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
             });
         });
 

src/apps.js

@@ -52,9 +52,10 @@ var addons = require('./addons.js'),
     constants = require('./constants.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apps'),
-    docker = require('./docker.js').connection,
+    docker = require('./docker.js'),
     fs = require('fs'),
     manifestFormat = require('cloudron-manifestformat'),
+    once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
@@ -72,6 +73,8 @@ var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
     RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
 
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
 
@@ -173,7 +176,7 @@ function validatePortBindings(portBindings, tcpPorts) {
         if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
         if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
 
-        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, + portBindings[env]);
+        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, String(portBindings[env]));
     }
 
     // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
@@ -290,6 +293,7 @@ function purchase(appStoreId, callback) {
     superagent.post(url).query({ token: config.token() }).end(function (error, res) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
         if (res.status === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+        if (res.status === 404) return callback(new AppsError(AppsError.NOT_FOUND));
         if (res.status !== 201 && res.status !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
 
         callback(null);
@@ -493,8 +497,6 @@ function getLogs(appId, lines, follow, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
-
         var args = [ '--output=json', '--no-pager', '--lines=' + lines ];
         if (follow) args.push('--follow');
         args = args.concat(appLogFilter(app));
@@ -511,7 +513,7 @@ function getLogs(appId, lines, follow, callback) {
                 monotonicTimestamp: obj.__MONOTONIC_TIMESTAMP,
                 message: obj.MESSAGE,
                 source: source || 'main'
-            });
+            }) + '\n';
         });
 
         transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process
@@ -643,31 +645,42 @@ function exec(appId, options, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var container = docker.getContainer(app.containerId);
-
-       var execOptions = {
+       var createOptions = {
             AttachStdin: true,
             AttachStdout: true,
             AttachStderr: true,
-            Tty: true,
-            Cmd: cmd
+            OpenStdin: true,
+            StdinOnce: false,
+            Tty: true
         };
 
-        container.exec(execOptions, function (error, exec) {
+        docker.createSubcontainer(app, app.id + '-exec-' + Date.now(), cmd, createOptions, function (error, container) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-            var startOptions = {
-                Detach: false,
-                Tty: true,
-                stdin: true // this is a dockerode option that enabled openStdin in the modem
-            };
-            exec.start(startOptions, function(error, stream) {
+
+            container.attach({ stream: true, stdin: true, stdout: true, stderr: true }, function (error, stream) {
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                if (options.rows && options.columns) {
-                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
-                }
+                docker.startContainer(container.id, function (error) {
+                    if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-                return callback(null, stream);
+                    if (options.rows && options.columns) {
+                        container.resize({ h: options.rows, w: options.columns }, NOOP_CALLBACK);
+                    }
+
+                    var deleteContainer = once(docker.deleteContainer.bind(null, container.id, NOOP_CALLBACK));
+
+                    container.wait(function (error) {
+                        if (error) debug('Error waiting on container', error);
+
+                        debug('exec: container finished', container.id);
+
+                        deleteContainer();
+                    });
+
+                    stream.close = deleteContainer;
+
+                    callback(null, stream);
+                });
             });
         });
     });

src/apptask.js

@@ -660,7 +660,7 @@ function stopApp(app, callback) {
     docker.stopContainers(app.id, function (error) {
         if (error) return callback(error);
 
-        updateApp(app, { runState: appdb.RSTATE_STOPPED }, callback);
+        updateApp(app, { runState: appdb.RSTATE_STOPPED, health: null }, callback);
     });
 }
 

src/backups.js

@@ -16,7 +16,7 @@ var assert = require('assert'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
     s3 = require('./storage/s3.js'),
-    superagent = require('superagent'),
+    settings = require('./settings.js'),
     util = require('util');
 
 function BackupsError(reason, errorOrMessage) {
@@ -43,8 +43,12 @@ BackupsError.INTERNAL_ERROR = 'internal error';
 BackupsError.MISSING_CREDENTIALS = 'missing credentials';
 
 // choose which storage backend we use for test purpose we use s3
-function api() {
-    return config.token() ? caas : s3;
+function api(provider) {
+    switch (provider) {
+        case 'caas': return caas;
+        case 's3': return s3;
+        default: return null;
+    }
 }
 
 function getAllPaged(page, perPage, callback) {
@@ -52,15 +56,14 @@ function getAllPaged(page, perPage, callback) {
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    superagent.get(url).query({ token: config.token() }).end(function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !util.isArray(result.body.backups)) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
-        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
-        return callback(null, result.body.backups);
+            return callback(null, backups); // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first
+        });
     });
 }
 
@@ -75,19 +78,23 @@ function getBackupUrl(app, callback) {
         filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
     }
 
-    api().getSignedUploadUrl(filename, function (error, result) {
-        if (error) return callback(error);
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        var obj = {
-            id: filename,
-            url: result.url,
-            sessionToken: result.sessionToken,
-            backupKey: config.backupKey()
-        };
+        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+            if (error) return callback(error);
 
-        debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            var obj = {
+                id: filename,
+                url: result.url,
+                sessionToken: result.sessionToken,
+                backupKey: backupConfig.key
+            };
 
-        callback(null, obj);
+            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+            callback(null, obj);
+        });
     });
 }
 
@@ -96,19 +103,23 @@ function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    api().getSignedDownloadUrl(backupId, function (error, result) {
-        if (error) return callback(error);
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-        var obj = {
-            id: backupId,
-            url: result.url,
-            sessionToken: result.sessionToken,
-            backupKey: config.backupKey()
-        };
+        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
+            if (error) return callback(error);
 
-        debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+            var obj = {
+                id: backupId,
+                url: result.url,
+                sessionToken: result.sessionToken,
+                backupKey: backupConfig.key
+            };
 
-        callback(null, obj);
+            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+            callback(null, obj);
+        });
     });
 }
 
@@ -118,9 +129,14 @@ function copyLastBackup(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    api().copyObject(app.lastBackupId, toFilename, function (error) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
 
-        return callback(null, toFilename);
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilename, function (error) {
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+            return callback(null, toFilename);
+        });
     });
 }

src/cloudron.js

@@ -243,6 +243,8 @@ function getStatus(callback) {
             callback(null, {
                 activated: count !== 0,
                 version: config.version(),
+                boxVersionsUrl: config.get('boxVersionsUrl'),
+                apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool
                 cloudronName: cloudronName
             });
         });
@@ -272,7 +274,7 @@ function getConfig(callback) {
 
     getCloudronDetails(function (error, result) {
         if (error) {
-            console.error('Failed to fetch cloudron details.', error);
+            debug('Failed to fetch cloudron details.', error);
 
             // set fallback values to avoid dependency on appstore
             result = {
@@ -411,10 +413,11 @@ function addDnsRecords() {
 
             debug('addDnsRecords: will update %j', records);
 
-            async.eachSeries(records, function (record, iteratorCallback) {
+            async.mapSeries(records, function (record, iteratorCallback) {
                 subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
-            }, function (error) {
+            }, function (error, changeIds) {
                 if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
+                else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
 
                 retryCallback(error);
             });
@@ -489,7 +492,7 @@ function update(boxUpdateInfo, callback) {
         debug('Starting upgrade');
         doUpgrade(boxUpdateInfo, function (error) {
             if (error) {
-                debug('Upgrade failed with error: %s', error);
+                console.error('Upgrade failed with error:', error);
                 locker.unlock(locker.OP_BOX_UPDATE);
             }
         });
@@ -497,7 +500,7 @@ function update(boxUpdateInfo, callback) {
         debug('Starting update');
         doUpdate(boxUpdateInfo, function (error) {
             if (error) {
-                debug('Update failed with error: %s', error);
+                console.error('Update failed with error:', error);
                 locker.unlock(locker.OP_BOX_UPDATE);
             }
         });
@@ -562,19 +565,31 @@ function doUpdate(boxUpdateInfo, callback) {
 
                 // this data is opaque to the installer
                 data: {
-                    apiServerOrigin: config.apiServerOrigin(),
-                    aws: config.aws(),
-                    backupKey: config.backupKey(),
-                    boxVersionsUrl: config.get('boxVersionsUrl'),
-                    fqdn: config.fqdn(),
-                    isCustomDomain: config.isCustomDomain(),
-                    restoreUrl: null,
-                    restoreKey: null,
                     token: config.token(),
+                    apiServerOrigin: config.apiServerOrigin(),
+                    webServerOrigin: config.webServerOrigin(),
+                    fqdn: config.fqdn(),
                     tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
                     tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    isCustomDomain: config.isCustomDomain(),
+
+                    appstore: {
+                        token: config.token(),
+                        apiServerOrigin: config.apiServerOrigin()
+                    },
+                    caas: {
+                        token: config.token(),
+                        apiServerOrigin: config.apiServerOrigin(),
+                        webServerOrigin: config.webServerOrigin()
+                    },
+                    tlsConfig: {
+                        provider: 'caas',
+                        cert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                        key: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    },
+
                     version: boxUpdateInfo.version,
-                    webServerOrigin: config.webServerOrigin()
+                    boxVersionsUrl: config.get('boxVersionsUrl')
                 }
             };
 
@@ -685,13 +700,13 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
                 if (error && error.reason !== AppsError.BAD_STATE) {
                     debugApp(app, 'Unable to backup', error);
                     return iteratorCallback(error);
                 }
 
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
                 iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
             });
         }, function appsBackedUp(error, backupIds) {

src/config.js

@@ -33,9 +33,6 @@ exports = module.exports = {
 
     isDev: isDev,
 
-    backupKey: backupKey,
-    aws: aws,
-
     // for testing resets to defaults
     _reset: initConfig
 };
@@ -84,13 +81,6 @@ function initConfig() {
     data.ldapPort = 3002;
     data.oauthProxyPort = 3003;
     data.simpleAuthPort = 3004;
-    data.backupKey = 'backupKey';
-    data.aws = {
-        backupBucket: null,
-        backupPrefix: null,
-        accessKeyId: null,      // selfhosting only
-        secretAccessKey: null   // selfhosting only
-    };
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -107,9 +97,6 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
-        data.aws.backupBucket = 'testbucket';
-        data.aws.backupPrefix = 'testprefix';
-        data.aws.endpoint = 'http://localhost:5353';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -202,11 +189,3 @@ function database() {
 function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
-
-function backupKey() {
-    return get('backupKey');
-}
-
-function aws() {
-    return get('aws');
-}

src/database.js

@@ -126,6 +126,8 @@ function clear(callback) {
 function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+
     gConnectionPool.getConnection(function (error, connection) {
         if (error) return callback(error);
 

src/dns/caas.js

@@ -18,7 +18,8 @@ var assert = require('assert'),
     util = require('util'),
     _ = require('underscore');
 
-function add(zoneName, subdomain, type, values, callback) {
+function add(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
@@ -36,7 +37,7 @@ function add(zoneName, subdomain, type, values, callback) {
 
     superagent
         .post(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
-        .query({ token: config.token() })
+        .query({ token: dnsConfig.token })
         .send(data)
         .end(function (error, result) {
             if (error) return callback(error);
@@ -47,7 +48,8 @@ function add(zoneName, subdomain, type, values, callback) {
         });
 }
 
-function get(zoneName, subdomain, type, callback) {
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
@@ -59,7 +61,7 @@ function get(zoneName, subdomain, type, callback) {
 
     superagent
         .get(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
-        .query({ token: config.token(), type: type })
+        .query({ token: dnsConfig.token, type: type })
         .end(function (error, result) {
             if (error) return callback(error);
             if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
@@ -68,23 +70,25 @@ function get(zoneName, subdomain, type, callback) {
         });
 }
 
-function update(zoneName, subdomain, type, values, callback) {
+function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    get(zoneName, subdomain, type, function (error, result) {
+    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
         if (error) return callback(error);
 
         if (_.isEqual(values, result)) return callback();
 
-        add(zoneName, subdomain, type, values, callback);
+        add(dnsConfig, zoneName, subdomain, type, values, callback);
     });
 }
 
-function del(zoneName, subdomain, type, values, callback) {
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
@@ -100,7 +104,7 @@ function del(zoneName, subdomain, type, values, callback) {
 
     superagent
         .del(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
-        .query({ token: config.token() })
+        .query({ token: dnsConfig.token })
         .send(data)
         .end(function (error, result) {
             if (error) return callback(error);
@@ -112,7 +116,8 @@ function del(zoneName, subdomain, type, values, callback) {
         });
 }
 
-function getChangeStatus(changeId, callback) {
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof changeId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
@@ -120,7 +125,7 @@ function getChangeStatus(changeId, callback) {
 
     superagent
         .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
-        .query({ token: config.token() })
+        .query({ token: dnsConfig.token })
         .end(function (error, result) {
             if (error) return callback(error);
             if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));

src/dns/route53.js

@@ -14,52 +14,45 @@ var assert = require('assert'),
     AWS = require('aws-sdk'),
     config = require('../config.js'),
     debug = require('debug')('box:dns/route53'),
-    settings = require('../settings.js'),
     SubdomainError = require('../subdomains.js').SubdomainError,
     util = require('util'),
     _ = require('underscore');
 
-function getDnsCredentials(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function getDnsCredentials(dnsConfig) {
+    assert.strictEqual(typeof dnsConfig, 'object');
 
-    settings.getDnsConfig(function (error, dnsConfig) {
-        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+    var credentials = {
+        accessKeyId: dnsConfig.accessKeyId,
+        secretAccessKey: dnsConfig.secretAccessKey,
+        region: dnsConfig.region
+    };
 
-        var credentials = {
-            accessKeyId: dnsConfig.accessKeyId,
-            secretAccessKey: dnsConfig.secretAccessKey,
-            region: dnsConfig.region
-        };
+    if (dnsConfig.endpoint) credentials.endpoint = new AWS.Endpoint(dnsConfig.endpoint);
 
-        if (dnsConfig.endpoint) credentials.endpoint = new AWS.Endpoint(dnsConfig.endpoint);
-
-        callback(null, credentials);
-    });
+    return credentials;
 }
 
-function getZoneByName(zoneName, callback) {
+function getZoneByName(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getDnsCredentials(function (error, credentials) {
-        if (error) return callback(error);
+    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    route53.listHostedZones({}, function (error, result) {
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
 
-        var route53 = new AWS.Route53(credentials);
-        route53.listHostedZones({}, function (error, result) {
-            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+        var zone = result.HostedZones.filter(function (zone) {
+            return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+        })[0];
 
-            var zone = result.HostedZones.filter(function (zone) {
-                return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
-            })[0];
+        if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
 
-            if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
-
-            callback(null, zone);
-        });
+        callback(null, zone);
     });
 }
 
-function add(zoneName, subdomain, type, values, callback) {
+function add(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
@@ -68,7 +61,7 @@ function add(zoneName, subdomain, type, values, callback) {
 
     debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
 
-    getZoneByName(zoneName, function (error, zone) {
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
         var fqdn = config.appFqdn(subdomain);
@@ -89,46 +82,44 @@ function add(zoneName, subdomain, type, values, callback) {
             HostedZoneId: zone.Id
         };
 
-        getDnsCredentials(function (error, credentials) {
-            if (error) return callback(error);
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'PriorRequestNotComplete') {
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            } else if (error) {
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+            }
 
-            var route53 = new AWS.Route53(credentials);
-            route53.changeResourceRecordSets(params, function(error, result) {
-                if (error && error.code === 'PriorRequestNotComplete') {
-                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
-                } else if (error) {
-                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
-                }
-
-                callback(null, result.ChangeInfo.Id);
-            });
+            callback(null, result.ChangeInfo.Id);
         });
     });
 }
 
-function update(zoneName, subdomain, type, values, callback) {
+function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    get(zoneName, subdomain, type, function (error, result) {
+    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
         if (error) return callback(error);
 
         if (_.isEqual(values, result)) return callback();
 
-        add(zoneName, subdomain, type, values, callback);
+        add(dnsConfig, zoneName, subdomain, type, values, callback);
     });
 }
 
-function get(zoneName, subdomain, type, callback) {
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getZoneByName(zoneName, function (error, zone) {
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
         var params = {
@@ -138,31 +129,28 @@ function get(zoneName, subdomain, type, callback) {
             StartRecordType: type
         };
 
-        getDnsCredentials(function (error, credentials) {
-            if (error) return callback(error);
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.listResourceRecordSets(params, function (error, result) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
+            if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
 
-            var route53 = new AWS.Route53(credentials);
-            route53.listResourceRecordSets(params, function (error, result) {
-                if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
-                if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
-                if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
+            var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
 
-                var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
-
-                callback(null, values);
-            });
+            callback(null, values);
         });
     });
 }
 
-function del(zoneName, subdomain, type, values, callback) {
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof subdomain, 'string');
     assert.strictEqual(typeof type, 'string');
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    getZoneByName(zoneName, function (error, zone) {
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
         if (error) return callback(error);
 
         var fqdn = config.appFqdn(subdomain);
@@ -185,48 +173,42 @@ function del(zoneName, subdomain, type, values, callback) {
             HostedZoneId: zone.Id
         };
 
-        getDnsCredentials(function (error, credentials) {
-            if (error) return callback(error);
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                debug('delSubdomain: resource record set not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+            } else if (error && error.code === 'NoSuchHostedZone') {
+                debug('delSubdomain: hosted zone not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+            } else if (error && error.code === 'PriorRequestNotComplete') {
+                debug('delSubdomain: resource is still busy', error);
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+            } else if (error && error.code === 'InvalidChangeBatch') {
+                debug('delSubdomain: invalid change batch. No such record to be deleted.');
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+            } else if (error) {
+                debug('delSubdomain: error', error);
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+            }
 
-            var route53 = new AWS.Route53(credentials);
-            route53.changeResourceRecordSets(params, function(error, result) {
-                if (error && error.message && error.message.indexOf('it was not found') !== -1) {
-                    debug('delSubdomain: resource record set not found.', error);
-                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
-                } else if (error && error.code === 'NoSuchHostedZone') {
-                    debug('delSubdomain: hosted zone not found.', error);
-                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
-                } else if (error && error.code === 'PriorRequestNotComplete') {
-                    debug('delSubdomain: resource is still busy', error);
-                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
-                } else if (error && error.code === 'InvalidChangeBatch') {
-                    debug('delSubdomain: invalid change batch. No such record to be deleted.');
-                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
-                } else if (error) {
-                    debug('delSubdomain: error', error);
-                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
-                }
-
-                callback(null);
-            });
+            callback(null);
         });
     });
 }
 
-function getChangeStatus(changeId, callback) {
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof changeId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     if (changeId === '') return callback(null, 'INSYNC');
 
-    getDnsCredentials(function (error, credentials) {
+    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    route53.getChange({ Id: changeId }, function (error, result) {
         if (error) return callback(error);
 
-        var route53 = new AWS.Route53(credentials);
-        route53.getChange({ Id: changeId }, function (error, result) {
-            if (error) return callback(error);
-
-            callback(null, result.ChangeInfo.Status);
-        });
+        callback(null, result.ChangeInfo.Status);
     });
 }
+

src/docker.js

@@ -8,7 +8,8 @@ var addons = require('./addons.js'),
     Docker = require('dockerode'),
     safe = require('safetydance'),
     semver = require('semver'),
-    util = require('util');
+    util = require('util'),
+    _ = require('underscore');
 
 exports = module.exports = {
     connection: connectionInstance(),
@@ -64,18 +65,17 @@ function pullImage(manifest, callback) {
         // is emitted as a chunk
         stream.on('data', function (chunk) {
             var data = safe.JSON.parse(chunk) || { };
-            debug('pullImage data: %j', data);
+            debug('pullImage %s: %j', manifest.id, data);
 
             // The information here is useless because this is per layer as opposed to per image
             if (data.status) {
-                // debugApp(app, 'progress: %s', data.status); // progressDetail { current, total }
             } else if (data.error) {
-                debug('pullImage error detail: %s', data.errorDetail.message);
+                debug('pullImage error %s: %s', manifest.id, data.errorDetail.message);
             }
         });
 
         stream.on('end', function () {
-            debug('downloaded image %s successfully', manifest.dockerImage);
+            debug('downloaded image %s of %s successfully', manifest.dockerImage, manifest.id);
 
             var image = docker.getImage(manifest.dockerImage);
 
@@ -84,14 +84,14 @@ function pullImage(manifest, callback) {
                 if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
                 if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
 
-                debug('This image exposes ports: %j', data.Config.ExposedPorts);
+                debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
 
                 callback(null);
             });
         });
 
         stream.on('error', function (error) {
-            debug('error pulling image %s : %j', manifest.dockerImage, error);
+            debug('error pulling image %s of %s: %j', manifest.dockerImage, manifest.id, error);
 
             callback(error);
         });
@@ -102,12 +102,12 @@ function downloadImage(manifest, callback) {
     assert.strictEqual(typeof manifest, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('downloadImage %s', manifest.dockerImage);
+    debug('downloadImage %s %s', manifest.id, manifest.dockerImage);
 
     var attempt = 1;
 
-    async.retry({ times: 5, interval: 15000 }, function (retryCallback) {
-        debug('Downloading image. attempt: %s', attempt++);
+    async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
+        debug('Downloading image %s %s. attempt: %s', manifest.id, manifest.dockerImage, attempt++);
 
         pullImage(manifest, function (error) {
             if (error) console.error(error);
@@ -117,10 +117,11 @@ function downloadImage(manifest, callback) {
     }, callback);
 }
 
-function createSubcontainer(app, name, cmd, callback) {
+function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
     assert(!cmd || util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var docker = exports.connection,
@@ -153,6 +154,9 @@ function createSubcontainer(app, name, cmd, callback) {
     }
 
     var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
+    // this means cloudron exec does not work
+    var isolatedNetworkNs = true;
 
     addons.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(new Error('Error getting addon environment : ' + error));
@@ -160,7 +164,8 @@ function createSubcontainer(app, name, cmd, callback) {
         var containerOptions = {
             name: name, // used for filtering logs
             // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
-            Hostname: semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location),
+            // for subcontainers, this should not be set because we already share the network namespace with app container
+            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
             Tty: isAppContainer,
             Image: app.manifest.dockerImage,
             Cmd: cmd,
@@ -182,28 +187,30 @@ function createSubcontainer(app, name, cmd, callback) {
                 PortBindings: isAppContainer ? dockerPortBindings : { },
                 PublishAllPorts: false,
                 ReadonlyRootfs: semver.gte(targetBoxVersion(app.manifest), '0.0.66'), // see also Volumes in startContainer
-                Links: addons.getLinksSync(app, app.manifest.addons),
                 RestartPolicy: {
                     "Name": isAppContainer ? "always" : "no",
                     "MaximumRetryCount": 0
                 },
                 CpuShares: 512, // relative to 1024 for system processes
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
+                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
+                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
                 SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
             }
         };
+        containerOptions = _.extend(containerOptions, options);
 
         // older versions wanted a writable /var/log
         if (semver.lte(targetBoxVersion(app.manifest), '0.0.71')) containerOptions.Volumes['/var/log'] = {};
 
-        debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
+        debugApp(app, 'Creating container for %s with options %j', app.manifest.dockerImage, containerOptions);
 
         docker.createContainer(containerOptions, callback);
     });
 }
 
 function createContainer(app, callback) {
-    createSubcontainer(app, app.id /* name */, null /* cmd */, callback);
+    createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback);
 }
 
 function startContainer(containerId, callback) {

src/mail_templates/app_down.ejs

@@ -5,8 +5,7 @@ Dear Admin,
 The application titled '<%= title %>' that you installed at <%= appFqdn %>
 is not responding.
 
-This is most likely a problem in the application. Please report this issue to
-support@cloudron.io (by forwarding this email).
+This is most likely a problem in the application.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mailer.js

@@ -133,6 +133,15 @@ function checkDns() {
 function processQueue() {
     assert(gDnsReady);
 
+    sendMails(gMailQueue);
+    gMailQueue = [ ];
+}
+
+// note : this function should NOT access the database. it is called by the crashnotifier
+// which does not initialize mailer or the databse
+function sendMails(queue) {
+    assert(util.isArray(queue));
+
     docker.getContainer('mail').inspect(function (error, data) {
         if (error) return console.error(error);
 
@@ -144,12 +153,9 @@ function processQueue() {
             port: 2500 // this value comes from mail container
         }));
 
-        var mailQueueCopy = gMailQueue;
-        gMailQueue = [ ];
+        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
 
-        debug('Processing mail queue of size %d (through %s:2500)', mailQueueCopy.length, mailServerIp);
-
-        async.mapSeries(mailQueueCopy, function iterator(mailOptions, callback) {
+        async.mapSeries(queue, function iterator(mailOptions, callback) {
             transport.sendMail(mailOptions, function (error) {
                 if (error) return console.error(error); // TODO: requeue?
                 debug('Email sent to ' + mailOptions.to);
@@ -278,7 +284,7 @@ function appDied(app) {
 
         var mailOptions = {
             from: config.get('adminEmail'),
-            to: adminEmails.join(', '),
+            to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
         };
@@ -323,6 +329,8 @@ function appUpdateAvailable(app, updateInfo) {
     });
 }
 
+// this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
+// crashnotifier should be able to send mail when there is no db
 function sendCrashNotification(program, context) {
     assert.strictEqual(typeof program, 'string');
     assert.strictEqual(typeof context, 'string');
@@ -334,7 +342,7 @@ function sendCrashNotification(program, context) {
         text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
     };
 
-    enqueue(mailOptions);
+    sendMails([ mailOptions ]);
 }
 
 function sendFeedback(user, type, subject, description) {

src/oauth2views/callback.ejs

@@ -35,9 +35,9 @@ args.forEach(function (arg) {
 });
 
 if (code && redirectURI) {
-    window.location.href = redirectURI + '?code=' + code + (state ? '&state=' + state : '');
+    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'code=' + code + (state ? '&state=' + state : '');
 } else if (redirectURI && accessToken) {
-    window.location.href = redirectURI + '?token=' + accessToken + (state ? '&state=' + state : '');
+    window.location.href = redirectURI + (redirectURI.indexOf('?') !== -1 ? '&' : '?') + 'token=' + accessToken + (state ? '&state=' + state : '');
 } else {
     window.location.href = '/api/v1/session/login';
 }

src/paths.js

@@ -28,5 +28,8 @@ exports = module.exports = {
     CLOUDRON_AVATAR_FILE: path.join(config.baseDir(), 'data/box/avatar.png'),
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
 
-    UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json')
+    UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
+
+    ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme.key')
 };

src/routes/apps.js

@@ -133,6 +133,7 @@ function installApp(req, res, next) {
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
         if (error) return next(new HttpError(500, error));
@@ -368,6 +369,8 @@ function exec(req, res, next) {
 
         duplexStream.pipe(res.socket);
         res.socket.pipe(duplexStream);
+
+        res.on('close', duplexStream.close);
     });
 }
 

src/routes/oauth2.js

@@ -359,7 +359,7 @@ var authorization = [
             var redirectPath = url.parse(redirectURI).path;
             var redirectOrigin = client.redirectURI;
 
-            callback(null, client, '/api/v1/session/callback?redirectURI=' + url.resolve(redirectOrigin, redirectPath));
+            callback(null, client, '/api/v1/session/callback?redirectURI=' + encodeURIComponent(url.resolve(redirectOrigin, redirectPath)));
         });
     }),
     function (req, res, next) {

src/routes/settings.js

@@ -15,6 +15,9 @@ exports = module.exports = {
     getDnsConfig: getDnsConfig,
     setDnsConfig: setDnsConfig,
 
+    getBackupConfig: getBackupConfig,
+    setBackupConfig: setBackupConfig,
+
     setCertificate: setCertificate,
     setAdminCertificate: setAdminCertificate
 };
@@ -111,6 +114,27 @@ function setDnsConfig(req, res, next) {
     });
 }
 
+function getBackupConfig(req, res, next) {
+    settings.getBackupConfig(function (error, config) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, config));
+    });
+}
+
+function setBackupConfig(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
+
+    settings.setBackupConfig(req.body, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
 // default fallback cert
 function setCertificate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');

src/routes/test/apps-test.js

@@ -146,12 +146,17 @@ function setup(done) {
 
                 callback(null);
             });
-        }, function (callback) {
+        },
+
+        function (callback) {
             token_1 = tokendb.generateToken();
 
             // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
             tokendb.add(token_1, tokendb.PREFIX_USER + USERNAME_1, 'test-client-id',  Date.now() + 100000, '*', callback);
-        }
+        },
+
+        settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
+        settings.setBackupConfig.bind(null, { provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
     ], done);
 }
 
@@ -590,8 +595,6 @@ describe('App installation', function () {
                 apiHockServer = http.createServer(apiHockInstance.handler).listen(port, callback);
             },
 
-            settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
-
             function (callback) {
                 awsHockInstance
                     .get('/2013-04-01/hostedzone')

src/routes/test/backups-test.js

@@ -13,6 +13,7 @@ var appdb = require('../../appdb.js'),
     expect = require('expect.js'),
     request = require('superagent'),
     server = require('../../server.js'),
+    settings = require('../../settings.js'),
     nock = require('nock'),
     userdb = require('../../userdb.js');
 
@@ -52,6 +53,10 @@ function setup(done) {
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
             appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, false /* oauthProxy */, callback);
+        },
+
+        function createSettings(callback) {
+            settings.setBackupConfig({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' }, callback);
         }
     ], done);
 }
@@ -70,7 +75,7 @@ describe('Backups API', function () {
 
     describe('get', function () {
         it('cannot get backups with appstore request failing', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=APPSTORE_TOKEN').reply(401, {});
+            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(401, {});
 
             request.get(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })
@@ -82,7 +87,7 @@ describe('Backups API', function () {
         });
 
         it('can get backups', function (done) {
-            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=APPSTORE_TOKEN').reply(200, { backups: ['foo', 'bar']});
+            var req = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/backups?token=BACKUP_TOKEN').reply(200, { backups: ['foo', 'bar']});
 
             request.get(SERVER_URL + '/api/v1/backups')
                    .query({ access_token: token })
@@ -119,7 +124,7 @@ describe('Backups API', function () {
 
         it('succeeds', function (done) {
             var scope = nock(config.apiServerOrigin())
-                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                        .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
                         .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             request.post(SERVER_URL + '/api/v1/backups')
@@ -141,4 +146,3 @@ describe('Backups API', function () {
         });
     });
 });
-

src/routes/test/cloudron-test.js

@@ -282,6 +282,19 @@ describe('Cloudron', function () {
                         callback();
                     });
                 },
+
+                function setupBackupConfig(callback) {
+                    request.post(SERVER_URL + '/api/v1/settings/backup_config')
+                           .send({ provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(error).to.not.be.ok();
+                        expect(result.statusCode).to.equal(200);
+
+                        callback();
+                    });
+                }
+
             ], done);
         });
 
@@ -341,7 +354,6 @@ describe('Cloudron', function () {
             });
         });
 
-
         it('fails with wrong region type', function (done) {
             request.post(SERVER_URL + '/api/v1/cloudron/migrate')
                    .send({ size: 'small', region: 4, password: PASSWORD })
@@ -355,7 +367,7 @@ describe('Cloudron', function () {
 
         it('fails when in wrong state', function (done) {
             var scope2 = nock(config.apiServerOrigin())
-                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
                     .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             var scope3 = nock(config.apiServerOrigin())
@@ -391,7 +403,6 @@ describe('Cloudron', function () {
             });
         });
 
-
         it('succeeds', function (done) {
             var scope1 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/migrate?token=APPSTORE_TOKEN', function (body) {
                 return body.size && body.region && body.restoreKey;
@@ -404,7 +415,7 @@ describe('Cloudron', function () {
                     .reply(200, { id: 'someid' });
 
             var scope3 = nock(config.apiServerOrigin())
-                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=APPSTORE_TOKEN')
+                    .post('/api/v1/boxes/' + config.fqdn() + '/awscredentials?token=BACKUP_TOKEN')
                     .reply(201, { credentials: { AccessKeyId: 'accessKeyId', SecretAccessKey: 'secretAccessKey', SessionToken: 'sessionToken' } });
 
             injectShellMock();

src/routes/test/start_addons.sh

@@ -66,8 +66,6 @@ start_mongodb() {
 }
 
 start_mail() {
-    local mongodb_vars="MONGODB_ROOT_PASSWORD=${root_password}"
-
     docker rm -f mail 2>/dev/null 1>&2 || true
 
     docker run -dP --name=mail -e DOMAIN_NAME="localhost" \

src/scheduler.js

@@ -181,7 +181,9 @@ function doTask(appId, taskName, callback) {
             debug('Creating createSubcontainer for %s/%s : %s', app.id, taskName, gState[appId].schedulerConfig[taskName].command);
 
             // NOTE: if you change container name here, fix addons.js to return correct container names
-            docker.createSubcontainer(app, app.id + '-' + taskName, [ '/bin/sh', '-c', gState[appId].schedulerConfig[taskName].command ], function (error, container) {
+            docker.createSubcontainer(app, app.id + '-' + taskName, [ '/bin/sh', '-c', gState[appId].schedulerConfig[taskName].command ], { } /* options */, function (error, container) {
+                if (error) return callback(error);
+
                 appState.containerIds[taskName] = container.id;
 
                 saveState(gState);

src/server.js

@@ -20,7 +20,6 @@ var assert = require('assert'),
     middleware = require('./middleware'),
     passport = require('passport'),
     path = require('path'),
-    paths = require('./paths.js'),
     routes = require('./routes/index.js'),
     taskmanager = require('./taskmanager.js');
 
@@ -161,6 +160,8 @@ function initializeExpressSync() {
     router.post('/api/v1/settings/cloudron_avatar',    settingsScope, multipart, routes.settings.setCloudronAvatar);
     router.get ('/api/v1/settings/dns_config',         settingsScope, routes.settings.getDnsConfig);
     router.post('/api/v1/settings/dns_config',         settingsScope, routes.settings.setDnsConfig);
+    router.get ('/api/v1/settings/backup_config',      settingsScope, routes.settings.getBackupConfig);
+    router.post('/api/v1/settings/backup_config',      settingsScope, routes.settings.setBackupConfig);
     router.post('/api/v1/settings/certificate',        settingsScope, routes.settings.setCertificate);
     router.post('/api/v1/settings/admin_certificate',  settingsScope, routes.settings.setAdminCertificate);
 

src/settings.js

@@ -23,6 +23,9 @@ exports = module.exports = {
     getDnsConfig: getDnsConfig,
     setDnsConfig: setDnsConfig,
 
+    getBackupConfig: getBackupConfig,
+    setBackupConfig: setBackupConfig,
+
     getDefaultSync: getDefaultSync,
     getAll: getAll,
 
@@ -35,6 +38,7 @@ exports = module.exports = {
     CLOUDRON_NAME_KEY: 'cloudron_name',
     DEVELOPER_MODE_KEY: 'developer_mode',
     DNS_CONFIG_KEY: 'dns_config',
+    BACKUP_CONFIG_KEY: 'backup_config',
 
     events: new (require('events').EventEmitter)()
 };
@@ -62,6 +66,7 @@ var gDefaults = (function () {
     result[exports.CLOUDRON_NAME_KEY] = 'Cloudron';
     result[exports.DEVELOPER_MODE_KEY] = false;
     result[exports.DNS_CONFIG_KEY] = { };
+    result[exports.BACKUP_CONFIG_KEY] = { };
 
     return result;
 })();
@@ -271,6 +276,34 @@ function setDnsConfig(dnsConfig, callback) {
     });
 }
 
+function getBackupConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settingsdb.get(exports.BACKUP_CONFIG_KEY, function (error, value) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.BACKUP_CONFIG_KEY]);
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        callback(null, JSON.parse(value)); // provider, token, key, region, prefix, bucket
+    });
+}
+
+function setBackupConfig(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (backupConfig.provider !== 'caas') {
+        return callback(new SettingsError(SettingsError.BAD_FIELD, 'provider must be caas'));
+    }
+
+    settingsdb.set(exports.BACKUP_CONFIG_KEY, JSON.stringify(backupConfig), function (error) {
+        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
+
+        exports.events.emit(exports.BACKUP_CONFIG_KEY, backupConfig);
+
+        callback(null);
+    });
+}
+
 function getDefaultSync(name) {
     assert.strictEqual(typeof name, 'string');
 
@@ -338,13 +371,13 @@ function setCertificate(cert, key, callback) {
     var error = validateCertificate(cert, key, '*.' + config.fqdn());
     if (error) return callback(new SettingsError(SettingsError.INVALID_CERT, error.message));
 
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), cert)) {
-        return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
-    }
+    // backup the cert
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.cert'), cert)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.key'), key)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
 
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) {
-        return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
-    }
+    // copy over fallback cert
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), cert)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
 
     shell.sudo('setCertificate', [ RELOAD_NGINX_CMD ], function (error) {
         if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
@@ -367,6 +400,7 @@ function setAdminCertificate(cert, key, callback) {
     var error = validateCertificate(cert, key, vhost);
     if (error) return callback(new SettingsError(SettingsError.INVALID_CERT, error.message));
 
+    // backup the cert
     if (!safe.fs.writeFileSync(certFilePath, cert)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
     if (!safe.fs.writeFileSync(keyFilePath, key)) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, safe.error.message));
 

src/storage/caas.js

@@ -6,20 +6,24 @@ exports = module.exports = {
     getSignedUploadUrl: getSignedUploadUrl,
     getSignedDownloadUrl: getSignedDownloadUrl,
 
-    copyObject: copyObject
+    copyObject: copyObject,
+
+    getAllPaged: getAllPaged
 };
 
 var assert = require('assert'),
     AWS = require('aws-sdk'),
     config = require('../config.js'),
-    superagent = require('superagent');
+    superagent = require('superagent'),
+    util = require('util');
 
-function getBackupCredentials(callback) {
+function getBackupCredentials(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
-    assert(config.token());
+    assert(backupConfig.token);
 
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
-    superagent.post(url).query({ token: config.token() }).end(function (error, result) {
+    superagent.post(url).query({ token: backupConfig.token }).end(function (error, result) {
         if (error) return callback(error);
         if (result.statusCode !== 201) return callback(new Error(result.text));
         if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
@@ -31,24 +35,44 @@ function getBackupCredentials(callback) {
             region: 'us-east-1'
         };
 
-        if (config.aws().endpoint) credentials.endpoint = new AWS.Endpoint(config.aws().endpoint);
+        if (backupConfig.endpoint) credentials.endpoint = new AWS.Endpoint(backupConfig.endpoint);
 
         callback(null, credentials);
     });
 }
 
-function getSignedUploadUrl(filename, callback) {
+function getAllPaged(backupConfig, page, perPage, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+    superagent.get(url).query({ token: backupConfig.token }).end(function (error, result) {
+        if (error) return callback(error);
+        if (result.statusCode !== 200) return callback(new Error(result.text));
+        if (!result.body || !util.isArray(result.body.backups)) return callback(new Error('Unexpected response'));
+
+        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
+        return callback(null, result.body.backups);
+    });
+}
+
+function getSignedUploadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getBackupCredentials(function (error, credentials) {
+    if (!backupConfig.bucket || !backupConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var s3 = new AWS.S3(credentials);
 
         var params = {
-            Bucket: config.aws().backupBucket,
-            Key: config.aws().backupPrefix + '/' + filename,
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
             Expires: 60 * 30 /* 30 minutes */
         };
 
@@ -58,18 +82,21 @@ function getSignedUploadUrl(filename, callback) {
     });
 }
 
-function getSignedDownloadUrl(filename, callback) {
+function getSignedDownloadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getBackupCredentials(function (error, credentials) {
+    if (!backupConfig.bucket || !backupConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var s3 = new AWS.S3(credentials);
 
         var params = {
-            Bucket: config.aws().backupBucket,
-            Key: config.aws().backupPrefix + '/' + filename,
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
             Expires: 60 * 30 /* 30 minutes */
         };
 
@@ -79,18 +106,21 @@ function getSignedDownloadUrl(filename, callback) {
     });
 }
 
-function copyObject(from, to, callback) {
+function copyObject(backupConfig, from, to, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof from, 'string');
     assert.strictEqual(typeof to, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getBackupCredentials(function (error, credentials) {
+    if (!backupConfig.bucket || !backupConfig.prefix) return new Error('Invalid configuration'); // prevent error in s3
+
+    getBackupCredentials(backupConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var params = {
-            Bucket: config.aws().backupBucket, // target bucket
-            Key: config.aws().backupPrefix + '/' + to, // target file
-            CopySource: config.aws().backupBucket + '/' + config.aws().backupPrefix + '/' + from, // source
+            Bucket: backupConfig.bucket, // target bucket
+            Key: backupConfig.prefix + '/' + to, // target file
+            CopySource: backupConfig.bucket + '/' + backupConfig.prefix + '/' + from, // source
         };
 
         var s3 = new AWS.S3(credentials);

src/storage/s3.js

@@ -6,41 +6,52 @@ exports = module.exports = {
     getSignedUploadUrl: getSignedUploadUrl,
     getSignedDownloadUrl: getSignedDownloadUrl,
 
-    copyObject: copyObject
+    copyObject: copyObject,
+    getAllPaged: getAllPaged
 };
 
 var assert = require('assert'),
-    AWS = require('aws-sdk'),
-    config = require('../config.js');
+    AWS = require('aws-sdk');
 
-function getBackupCredentials(callback) {
+function getBackupCredentials(backupConfig, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!config.aws().accessKeyId || !config.aws().secretAccessKey) return callback(new Error('missing credentials'));
+    assert(backupConfig.accessKeyId && backupConfig.secretAccessKey);
 
     var credentials = {
-        accessKeyId: config.aws().accessKeyId,
-        secretAccessKey: config.aws().secretAccessKey,
+        accessKeyId: backupConfig.accessKeyId,
+        secretAccessKey: backupConfig.secretAccessKey,
         region: 'us-east-1'
     };
 
-    if (config.aws().endpoint) credentials.endpoint = new AWS.Endpoint(config.aws().endpoint);
+    if (backupConfig.endpoint) credentials.endpoint = new AWS.Endpoint(backupConfig.endpoint);
 
     callback(null, credentials);
 }
 
-function getSignedUploadUrl(filename, callback) {
+function getAllPaged(backupConfig, page, perPage, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    return callback(new Error('Not implemented yet'));
+}
+
+function getSignedUploadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getBackupCredentials(function (error, credentials) {
+    getBackupCredentials(backupConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var s3 = new AWS.S3(credentials);
 
         var params = {
-            Bucket: config.aws().backupBucket,
-            Key: config.aws().backupPrefix + '/' + filename,
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
             Expires: 60 * 30 /* 30 minutes */
         };
 
@@ -50,18 +61,19 @@ function getSignedUploadUrl(filename, callback) {
     });
 }
 
-function getSignedDownloadUrl(filename, callback) {
+function getSignedDownloadUrl(backupConfig, filename, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof filename, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getBackupCredentials(function (error, credentials) {
+    getBackupCredentials(backupConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var s3 = new AWS.S3(credentials);
 
         var params = {
-            Bucket: config.aws().backupBucket,
-            Key: config.aws().backupPrefix + '/' + filename,
+            Bucket: backupConfig.bucket,
+            Key: backupConfig.prefix + '/' + filename,
             Expires: 60 * 30 /* 30 minutes */
         };
 
@@ -71,18 +83,19 @@ function getSignedDownloadUrl(filename, callback) {
     });
 }
 
-function copyObject(from, to, callback) {
+function copyObject(backupConfig, from, to, callback) {
+    assert.strictEqual(typeof backupConfig, 'object');
     assert.strictEqual(typeof from, 'string');
     assert.strictEqual(typeof to, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    getBackupCredentials(function (error, credentials) {
+    getBackupCredentials(backupConfig, function (error, credentials) {
         if (error) return callback(error);
 
         var params = {
-            Bucket: config.aws().backupBucket, // target bucket
-            Key: config.aws().backupPrefix + '/' + to, // target file
-            CopySource: config.aws().backupBucket + '/' + config.aws().backupPrefix + '/' + from, // source
+            Bucket: backupConfig.bucket, // target bucket
+            Key: backupConfig.prefix + '/' + to, // target file
+            CopySource: backupConfig.bucket + '/' + backupConfig.prefix + '/' + from, // source
         };
 
         var s3 = new AWS.S3(credentials);

src/subdomains.js

@@ -16,6 +16,7 @@ var assert = require('assert'),
     caas = require('./dns/caas.js'),
     config = require('./config.js'),
     route53 = require('./dns/route53.js'),
+    settings = require('./settings.js'),
     util = require('util');
 
 function SubdomainError(reason, errorOrMessage) {
@@ -42,10 +43,17 @@ SubdomainError.NOT_FOUND = 'No such domain';
 SubdomainError.EXTERNAL_ERROR = 'External error';
 SubdomainError.STILL_BUSY = 'Still busy';
 SubdomainError.MISSING_CREDENTIALS = 'Missing credentials';
+SubdomainError.INTERNAL_ERROR = 'Missing credentials';
 
 // choose which subdomain backend we use for test purpose we use route53
-function api() {
-    return config.isCustomDomain() || config.TEST ? route53 : caas;
+function api(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
+    switch (provider) {
+        case 'caas': return caas;
+        case 'route53': return route53;
+        default: return null;
+    }
 }
 
 function add(subdomain, type, values, callback) {
@@ -54,9 +62,13 @@ function add(subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    api().add(config.zoneName(), subdomain, type, values, function (error, changeId) {
-        if (error) return callback(error);
-        callback(null, changeId);
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).add(dnsConfig, config.zoneName(), subdomain, type, values, function (error, changeId) {
+            if (error) return callback(error);
+            callback(null, changeId);
+        });
     });
 }
 
@@ -65,10 +77,14 @@ function get(subdomain, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    api().get(config.zoneName(), subdomain, type, function (error, values) {
-        if (error) return callback(error);
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
 
-        callback(null, values);
+        api(dnsConfig.provider).get(dnsConfig, config.zoneName(), subdomain, type, function (error, values) {
+            if (error) return callback(error);
+
+            callback(null, values);
+        });
     });
 }
 
@@ -78,10 +94,14 @@ function update(subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    api().update(config.zoneName(), subdomain, type, values, function (error) {
-        if (error) return callback(error);
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
 
-        callback(null);
+        api(dnsConfig.provider).update(dnsConfig, config.zoneName(), subdomain, type, values, function (error) {
+            if (error) return callback(error);
+
+            callback(null);
+        });
     });
 }
 
@@ -91,10 +111,14 @@ function remove(subdomain, type, values, callback) {
     assert(util.isArray(values));
     assert.strictEqual(typeof callback, 'function');
 
-    api().del(config.zoneName(), subdomain, type, values, function (error) {
-        if (error && error.reason !== SubdomainError.NOT_FOUND) return callback(error);
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
 
-        callback(null);
+        api(dnsConfig.provider).del(dnsConfig, config.zoneName(), subdomain, type, values, function (error) {
+            if (error && error.reason !== SubdomainError.NOT_FOUND) return callback(error);
+
+            callback(null);
+        });
     });
 }
 
@@ -102,8 +126,12 @@ function status(changeId, callback) {
     assert.strictEqual(typeof changeId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    api().getChangeStatus(changeId, function (error, status) {
-        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error));
-        callback(null, status === 'INSYNC' ? 'done' : 'pending');
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(new SubdomainError(SubdomainError.INTERNAL_ERROR, error));
+
+        api(dnsConfig.provider).getChangeStatus(dnsConfig, changeId, function (error, status) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error));
+            callback(null, status === 'INSYNC' ? 'done' : 'pending');
+        });
     });
 }

src/taskmanager.js

@@ -9,6 +9,7 @@ exports = module.exports = {
 
 var appdb = require('./appdb.js'),
     assert = require('assert'),
+    async = require('async'),
     child_process = require('child_process'),
     cloudron = require('./cloudron.js'),
     debug = require('debug')('box:taskmanager'),
@@ -39,14 +40,11 @@ function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     gPendingTasks = [ ]; // clear this first, otherwise stopAppTask will resume them
-    for (var appId in gActiveTasks) {
-        stopAppTask(appId);
-    }
 
     cloudron.events.removeListener(cloudron.EVENT_CONFIGURED, resumeTasks);
     locker.removeListener('unlocked', startNextTask);
 
-    callback(null);
+    async.eachSeries(Object.keys(gActiveTasks), stopAppTask, callback);
 }
 
 
@@ -95,8 +93,12 @@ function startAppTask(appId) {
     }
 
     gActiveTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
+
+    var pid = gActiveTasks[appId].pid;
+    debug('Started task of %s pid: %s', appId, pid);
+
     gActiveTasks[appId].once('exit', function (code) {
-        debug('Task for %s completed with status %s', appId, code);
+        debug('Task for %s pid %s completed with status %s', appId, pid, code);
         if (code && code !== 50) { // apptask crashed
             appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code }, NOOP_CALLBACK);
         }
@@ -105,21 +107,32 @@ function startAppTask(appId) {
     });
 }
 
-function stopAppTask(appId) {
+function stopAppTask(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     if (gActiveTasks[appId]) {
-        debug('stopAppTask : Killing existing task of %s with pid %s: ', appId, gActiveTasks[appId].pid);
+        debug('stopAppTask : Killing existing task of %s with pid %s', appId, gActiveTasks[appId].pid);
+        gActiveTasks[appId].once('exit', function () { callback(); });
         gActiveTasks[appId].kill(); // this will end up calling the 'exit' handler
-        delete gActiveTasks[appId];
-    } else if (gPendingTasks.indexOf(appId) !== -1) {
-        debug('stopAppTask: Removing existing pending task : %s', appId);
-        gPendingTasks = _.without(gPendingTasks, appId);
+        return;
     }
+
+    if (gPendingTasks.indexOf(appId) !== -1) {
+        debug('stopAppTask: Removing pending task : %s', appId);
+        gPendingTasks = _.without(gPendingTasks, appId);
+    } else {
+        debug('stopAppTask: no task for %s to be stopped', appId);
+    }
+
+    callback();
 }
 
-function restartAppTask(appId) {
-    stopAppTask(appId);
-    startAppTask(appId);
-}
+function restartAppTask(appId, callback) {
+    callback = callback || NOOP_CALLBACK;
 
+    async.series([
+        stopAppTask.bind(null, appId),
+        startAppTask.bind(null, appId)
+    ], callback);
+}

src/test/apptask-test.js

@@ -220,7 +220,7 @@ describe('apptask', function () {
     it('unregisters subdomain', function (done) {
         nock.cleanAll();
 
-        var awsScope = nock(config.aws().endpoint)
+        var awsScope = nock('http://localhost:5353')
             .get('/2013-04-01/hostedzone')
             .reply(200, js2xml('ListHostedZonesResponse', awsHostedZones, { arrayMap: { HostedZones: 'HostedZone'} }))
             .post('/2013-04-01/hostedzone/ZONEID/rrset/')

src/test/checkInstall

@@ -35,28 +35,40 @@ for script in "${scripts[@]}"; do
     fi
 done
 
+image_missing=""
+
 if ! docker inspect "${TEST_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull "${TEST_IMAGE}" for tests to run"
-    exit 1
+    echo "docker pull ${TEST_IMAGE}"
+    image_missing="true"
 fi
 
 if ! docker inspect "${REDIS_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${REDIS_IMAGE} for tests to run"
-    exit 1
+    echo "docker pull ${REDIS_IMAGE}"
+    image_missing="true"
 fi
 
 if ! docker inspect "${MYSQL_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${MYSQL_IMAGE} for tests to run"
-    exit 1
+    echo "docker pull ${MYSQL_IMAGE}"
+    image_missing="true"
 fi
 
 if ! docker inspect "${POSTGRESQL_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${POSTGRESQL_IMAGE} for tests to run"
-    exit 1
+    echo "docker pull ${POSTGRESQL_IMAGE}"
+    image_missing="true"
 fi
 
 if ! docker inspect "${MONGODB_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${MONGODB_IMAGE} for tests to run"
+    echo "docker pull ${MONGODB_IMAGE}"
+    image_missing="true"
+fi
+
+if ! docker inspect "${MAIL_IMAGE}" >/dev/null 2>/dev/null; then
+    echo "docker pull ${MAIL_IMAGE}"
+    image_missing="true"
+fi
+
+if [[ "${image_missing}" == "true" ]]; then
+    echo "Pull above images before running tests"
     exit 1
 fi
 

src/test/settings-test.js

@@ -100,6 +100,22 @@ describe('Settings', function () {
             });
         });
 
+        it('can set backup config', function (done) {
+            settings.setBackupConfig({ provider: 'caas', token: 'TOKEN' }, function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+
+        it('can get backup config', function (done) {
+            settings.getBackupConfig(function (error, dnsConfig) {
+                expect(error).to.be(null);
+                expect(dnsConfig.provider).to.be('caas');
+                expect(dnsConfig.token).to.be('TOKEN');
+                done();
+            });
+        });
+
         it('can get all values', function (done) {
             settings.getAll(function (error, allSettings) {
                 expect(error).to.be(null);

webadmin/src/index.html

@@ -147,7 +147,7 @@
                                 <li><a href="#/account"><i class="fa fa-user fa-fw"></i> Account</a></li>
                                 <li ng-show="user.admin"><a href="#/graphs"><i class="fa fa-bar-chart fa-fw"></i> Graphs</a></li>
                                 <li><a href="#/support"><i class="fa fa-comment fa-fw"></i> Support</a></li>
-                                <li ng-show="user.admin"><a href="#/certs"><i class="fa fa-certificate fa-fw"></i> DNS & Certs</a></li>
+                                <li ng-show="user.admin && config.isCustomDomain"><a href="#/certs"><i class="fa fa-certificate fa-fw"></i> DNS & Certs</a></li>
                                 <li ng-show="user.admin"><a href="#/settings"><i class="fa fa-wrench fa-fw"></i> Settings</a></li>
                                 <li class="divider"></li>
                                 <li><a href="" ng-click="logout($event)"><i class="fa fa-sign-out fa-fw"></i> Logout</a></li>

webadmin/src/js/setup.js

@@ -159,7 +159,7 @@ app.controller('StepController', ['$scope', '$route', '$location', 'Wizard', fun
         if (current === '/step1') {
             next = '/step2';
         } else if (current === '/step2') {
-            if (Wizard.dnsConfig.provider === 'caas') next = '/step4';
+            if (Wizard.dnsConfig === null) next = '/step4';
             else next = '/step3';
         } else if (current === '/step3') {
             next = '/step4';
@@ -210,7 +210,7 @@ app.controller('StepController', ['$scope', '$route', '$location', 'Wizard', fun
             image = null;
         };
         image.src = $scope.wizard.availableAvatars[randomIndex].data || $scope.wizard.availableAvatars[randomIndex].url;
-    } else if ($route.current.templateUrl === 'views/setup/step3.html' && Wizard.dnsConfig.provider === 'caas') {
+    } else if ($route.current.templateUrl === 'views/setup/step3.html' && Wizard.dnsConfig === null) {
         $location.path('/step4'); // not using custom domain
     }
 
@@ -229,6 +229,11 @@ app.controller('FinishController', ['$scope', '$location', 'Wizard', 'Client', f
         Client.changeCloudronAvatar($scope.wizard.avatarBlob, function (error) {
             if (error) return console.error('Unable to set avatar.', error);
 
+            if ($scope.wizard.dnsConfig === null) {
+                window.location.href = '/';
+                return;
+            }
+
             Client.setDnsConfig($scope.wizard.dnsConfig, function (error) {
                 if (error) return console.error('Unable to set dns config.', error);
 
@@ -256,12 +261,6 @@ app.controller('SetupController', ['$scope', '$location', 'Client', 'Wizard', fu
             accessKeyId: null,
             secretAccessKey: null
         };
-    } else {
-        Wizard.dnsConfig = {
-            provider: 'caas',
-            accessKeyId: '',
-            secretAccessKey: ''
-        };
     }
 
     Client.isServerFirstTime(function (error, isFirstTime) {

webadmin/src/views/certs.html

@@ -4,13 +4,13 @@
     </div>
 </div>
 
-<div style="max-width: 600px; margin: 0 auto;" ng-show="user.admin && config.isCustomDomain">
+<div style="max-width: 600px; margin: 0 auto;">
     <div class="text-left">
         <h3>DNS Credentials</h3>
     </div>
 </div>
 
-<div class="card" style="margin-bottom: 15px;" ng-show="user.admin && config.isCustomDomain">
+<div class="card" style="margin-bottom: 15px;">
     <div class="row">
         <div class="col-md-12">
             <p>Currently only Amazon <a href="https://aws.amazon.com/route53/">Route53</a> is supported. Let us know if you require a different DNS provider <a href="#/support">here</a>.</p>
@@ -53,13 +53,13 @@
     </div>
  </div>
 
-<div style="max-width: 600px; margin: 0 auto;" ng-show="user.admin && config.isCustomDomain">
+<div style="max-width: 600px; margin: 0 auto;">
     <div class="text-left">
         <h3>SSL Certificates</h3>
     </div>
 </div>
 
-<div class="card" style="margin-bottom: 15px;" ng-show="user.admin && config.isCustomDomain">
+<div class="card" style="margin-bottom: 15px;">
     <div class="row">
         <div class="col-md-12">
             <form name="defaultCertForm" ng-submit="setDefaultCert()">

webadmin/src/views/certs.js

@@ -1,7 +1,7 @@
 'use strict';
 
 angular.module('Application').controller('CertsController', ['$scope', '$location', 'Client', function ($scope, $location, Client) {
-    Client.onReady(function () { if (!Client.getUserInfo().admin) $location.path('/'); });
+    Client.onReady(function () { if (!Client.getUserInfo().admin || !Client.getConfig().isCustomDomain) $location.path('/'); });
 
     $scope.defaultCert = {
         error: null,