Fix redis hostname

Some apps like pasteboard try to curl the public app url from inside
the container. This fails because we set the hostname and the hostname
maps to the internal docker IP.

To fix this, simply export two environment variables providing the
app's domain and origin. The hostname is set to the app location instead
of the FQDN for debugging.

Fixes #521

.gitattributes

@@ -0,0 +1,6 @@
+# Skip files when using git archive
+.gitattributes export-ignore
+.gitignore export-ignore
+/scripts export-ignore
+test export-ignore
+

.gitignore

@@ -1 +1,9 @@
 node_modules/
+coverage/
+docs/
+webadmin/dist/
+setup/splash/website/
+
+# vim swap files
+*.swp
+

.jshintrc

@@ -1,9 +1,7 @@
 {
-    "node": true,
-    "browser": true,
-    "unused": true,
-    "multistr": true,
-    "globalstrict": true,
-    "predef": [ "angular", "$" ],
-    "esnext": true
+	"node": true,
+	"browser": true,
+	"unused": true,
+	"globalstrict": true,
+	"predef": [ "angular", "$" ]
 }

README.md

@@ -0,0 +1,10 @@
+The Box
+=======
+
+Development setup
+-----------------
+* sudo useradd -m yellowtent
+** Add admin-localhost as 127.0.0.1 in /etc/hosts
+** All apps will be installed as hypened-subdomains of localhost. You should add
+   hyphened-subdomains of your apps into /etc/hosts
+

box.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
+var appHealthMonitor = require('./src/apphealthmonitor.js'),
+    async = require('async'),
+    config = require('./src/config.js'),
+    ldap = require('./src/ldap.js'),
+    simpleauth = require('./src/simpleauth.js'),
+    oauthproxy = require('./src/oauthproxy.js'),
+    server = require('./src/server.js');
+
+console.log();
+console.log('==========================================');
+console.log(' Cloudron will use the following settings ');
+console.log('==========================================');
+console.log();
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
+console.log(' Version:                        ', config.version());
+console.log(' Admin Origin:                   ', config.adminOrigin());
+console.log(' Appstore token:                 ', config.token());
+console.log(' Appstore API server origin:     ', config.apiServerOrigin());
+console.log(' Appstore Web server origin:     ', config.webServerOrigin());
+console.log();
+console.log('==========================================');
+console.log();
+
+async.series([
+    server.start,
+    ldap.start,
+    simpleauth.start,
+    appHealthMonitor.start,
+    oauthproxy.start
+], function (error) {
+    if (error) {
+        console.error('Error starting server', error);
+        process.exit(1);
+    }
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});

crashnotifier.js

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var assert = require('assert'),
+    mailer = require('./src/mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
+
+function collectLogs(program, callback) {
+    assert.strictEqual(typeof program, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendCrashNotification(processName) {
+    collectLogs(processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending crash notification email for', processName);
+        mailer.sendCrashNotification(processName, result);
+    });
+}
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    mailer.initialize(function (error) {
+        if (error) return console.error(error);
+
+        sendCrashNotification(processName);
+    });
+}
+
+main();
+

docs/makeDocs

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eu
+
+./node_modules/.bin/apidoc -i src/routes -o docs

gulpfile.js

@@ -0,0 +1,159 @@
+/* jslint node:true */
+
+'use strict';
+
+var ejs = require('gulp-ejs'),
+    gulp = require('gulp'),
+    del = require('del'),
+    concat = require('gulp-concat'),
+    uglify = require('gulp-uglify'),
+    serve = require('gulp-serve'),
+    sass = require('gulp-sass'),
+    sourcemaps = require('gulp-sourcemaps'),
+    minifyCSS = require('gulp-minify-css'),
+    autoprefixer = require('gulp-autoprefixer'),
+    argv = require('yargs').argv;
+
+gulp.task('3rdparty', function () {
+    gulp.src([
+        'webadmin/src/3rdparty/**/*.js',
+        'webadmin/src/3rdparty/**/*.map',
+        'webadmin/src/3rdparty/**/*.css',
+        'webadmin/src/3rdparty/**/*.otf',
+        'webadmin/src/3rdparty/**/*.eot',
+        'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.ttf',
+        'webadmin/src/3rdparty/**/*.woff',
+        'webadmin/src/3rdparty/**/*.woff2'
+        ])
+        .pipe(gulp.dest('webadmin/dist/3rdparty/'))
+        .pipe(gulp.dest('setup/splash/website/3rdparty'));
+
+    gulp.src('node_modules/bootstrap-sass/assets/javascripts/bootstrap.min.js')
+        .pipe(gulp.dest('webadmin/dist/3rdparty/js'))
+        .pipe(gulp.dest('setup/splash/website/3rdparty/js'));
+});
+
+
+// --------------
+// JavaScript
+// --------------
+
+gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
+
+var oauth = {
+    clientId: argv.clientId || 'cid-webadmin',
+    clientSecret: argv.clientSecret || 'unused',
+    apiOrigin: argv.apiOrigin || ''
+};
+
+console.log();
+console.log('Using OAuth credentials:');
+console.log(' ClientId:     %s', oauth.clientId);
+console.log(' ClientSecret: %s', oauth.clientSecret);
+console.log(' Cloudron API: %s', oauth.apiOrigin || 'default');
+console.log();
+
+gulp.task('js-index', function () {
+    gulp.src([
+        'webadmin/src/js/index.js',
+        'webadmin/src/js/client.js',
+        'webadmin/src/js/appstore.js',
+        'webadmin/src/js/main.js',
+        'webadmin/src/views/*.js'
+        ])
+        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
+        .pipe(sourcemaps.init())
+        .pipe(concat('index.js', { newLine: ';' }))
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'));
+});
+
+gulp.task('js-setup', function () {
+    gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
+        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
+        .pipe(sourcemaps.init())
+        .pipe(concat('setup.js', { newLine: ';' }))
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'));
+});
+
+gulp.task('js-error', function () {
+    gulp.src(['webadmin/src/js/error.js'])
+        .pipe(sourcemaps.init())
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'));
+});
+
+gulp.task('js-update', function () {
+    gulp.src(['webadmin/src/js/update.js'])
+        .pipe(sourcemaps.init())
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'))
+        .pipe(gulp.dest('setup/splash/website/js'));
+});
+
+
+// --------------
+// HTML
+// --------------
+
+gulp.task('html', ['html-views', 'html-update'], function () {
+    return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
+});
+
+gulp.task('html-update', function () {
+    return gulp.src(['webadmin/src/update.html']).pipe(gulp.dest('setup/splash/website'));
+});
+
+gulp.task('html-views', function () {
+    return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
+});
+
+// --------------
+// CSS
+// --------------
+
+gulp.task('css', function () {
+    return gulp.src('webadmin/src/*.scss')
+        .pipe(sourcemaps.init())
+        .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
+        .pipe(autoprefixer())
+        .pipe(minifyCSS())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist'))
+        .pipe(gulp.dest('setup/splash/website'));
+});
+
+gulp.task('images', function () {
+    return gulp.src('webadmin/src/img/**')
+        .pipe(gulp.dest('webadmin/dist/img'));
+});
+
+// --------------
+// Utilities
+// --------------
+
+gulp.task('watch', ['default'], function () {
+    gulp.watch(['webadmin/src/*.scss'], ['css']);
+    gulp.watch(['webadmin/src/img/*'], ['images']);
+    gulp.watch(['webadmin/src/**/*.html'], ['html']);
+    gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
+    gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
+    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
+    gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
+    gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
+    gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);
+});
+
+gulp.task('clean', function () {
+    del.sync(['webadmin/dist', 'setup/splash/website']);
+});
+
+gulp.task('default', ['clean', 'html', 'js', '3rdparty', 'images', 'css'], function () {});
+
+gulp.task('develop', ['watch'], serve({ root: 'webadmin/dist', port: 4000 }));

index.js

@@ -1,60 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var dgram = require('dgram'),
-    fs = require('fs'),
-    path = require('path'),
-    mkdirp = require('mkdirp'),
-    parser = require('nsyslog-parser');
-
-const argv = require('yargs')
-    .default('port', 2514, 'The port to listen on')  // syslog is 514 so we prefix with 2
-    .default('logdir', '/tmp/logs', 'The root log directory')
-    .argv;
-
-const LOG_FILE_FOLDER = argv.logdir;
-const LOG_FILE_NAME = 'app.log';
-const PORT = argv.port;
-
-if (argv.version) {
-    console.log('1.0.1');
-    process.exit(0);
-}
-
-console.log();
-console.log('==========================================');
-console.log('          Cloudron Syslog Daemon          ');
-console.log('==========================================');
-console.log();
-console.log(' Log Folder:  ', LOG_FILE_FOLDER);
-console.log(' UDP Port:    ', PORT);
-console.log();
-console.log('==========================================');
-console.log();
-
-var server = dgram.createSocket('udp4');
-
-server.on('error', function (error) {
-    console.error('Error:', error);
-}).on('listening', function () {
-    console.log('Listening...');
-}).on('message', function (msg, rinfo) {
-    var info = parser(msg.toString());
-
-    if (!info || !info.appName) return console.log('Ignore unknown app log:', msg.toString());
-
-    // remove line breaks to avoid holes in the log file
-    // we do not ignore empty log lines, to allow gaps for potential ease of readability
-    const message = info.message.replace(/\n/g, '');
-
-    const filePath = path.join(LOG_FILE_FOLDER, info.appName);
-    const fileName = path.join(filePath, LOG_FILE_NAME);
-
-    try {
-        mkdirp.sync(filePath);
-        fs.appendFileSync(fileName, info.ts.toISOString() + ' ' + message + '\n');
-    } catch (error) {
-        console.error(error);
-    }
-}).bind(PORT);

migrations/20141021192552-db-create.js

@@ -0,0 +1,14 @@
+var dbm = require('db-migrate');
+var type = dbm.dataType;
+var url = require('url');
+
+exports.up = function(db, callback) {
+    var dbName = url.parse(process.env.DATABASE_URL).path.substr(1); // remove slash
+
+    // by default, mysql collates case insensitively. 'utf8_general_cs' is not available
+    db.runSql('ALTER DATABASE ' + dbName + '  DEFAULT CHARACTER SET=utf8 DEFAULT COLLATE utf8_bin', callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20141021192554-db-init.js

@@ -0,0 +1,19 @@
+var dbm = require('db-migrate');
+var type = dbm.dataType;
+
+var fs = require('fs'),
+    async = require('async'),
+    path = require('path');
+
+exports.up = function(db, callback) {
+    var schema = fs.readFileSync(path.join(__dirname, 'initial-schema.sql')).toString('utf8');
+    var statements = schema.split(';');
+    async.eachSeries(statements, function (statement, callback) {
+        if (statement.trim().length === 0) return callback(null);
+        db.runSql(statement, callback);
+    }, callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE users, tokens, clients, apps, appPortBindings, authcodes, settings', callback);
+};

migrations/20150303114527-users-add-resetToken.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN resetToken VARCHAR(128) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN resetToken', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150303160528-tokens-expires-bigint.js

@@ -0,0 +1,20 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM tokens', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE tokens MODIFY expires BIGINT', [], function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens MODIFY expires VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150303171203-authcodes-add-expiresAt.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE authcodes ADD COLUMN expiresAt BIGINT NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE authcodes DROP COLUMN expiresAt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150308054950-appportbindings-add-environmentVariable.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings ADD COLUMN environmentVariable VARCHAR(128) NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings DROP COLUMN environmentVariable', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150309023251-appportbindings-drop-containerport.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings DROP COLUMN containerPort', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings ADD COLUMN containerPort VARCHAR(5) NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150311120713-tokens-rename-userid-to-identifier.js

@@ -0,0 +1,20 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM tokens', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE tokens CHANGE userId identifier VARCHAR(128) NOT NULL', [], function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens CHANGE identifier userId VARCHAR(128) NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150323044254-apps-drop-version.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN version', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN version VARCHAR(32)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150323174906-apps-alter-health.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN healthy, ADD COLUMN health VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN health, ADD COLUMN healthy INTEGER', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150326072904-apps-add-lastBackupId.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupId VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupId', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150404093025-apps-add-createdAt.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN createdAt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150430165335-settings-default-autoupdatepattern.js

@@ -0,0 +1,12 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    // everyday at 1am
+    db.runSql('INSERT settings (name, value) VALUES("autoupdate_pattern", ?)', [ '00 00 1 * * *' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="autoupdate_pattern"', [ ], callback);
+}
+

migrations/20150430210225-settings-default-timezone.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+var safe = require('safetydance');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
+    tz = tz ? tz.trim() : 'America/Los_Angeles';
+
+    db.runSql('INSERT settings (name, value) VALUES("time_zone", ?)', [ tz ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="time_zone"', [ ], callback);
+};
+

migrations/20150615075901-users-add-unique-constraints.js

@@ -0,0 +1,24 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+
+    // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address
+
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_username UNIQUE (username)'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_email UNIQUE (email)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_username'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(512)'),
+        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_email'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(512)'),
+    ], callback);
+};

migrations/20150615134751-users-adjust-username-and-email-constraints.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
+    ], callback);
+};

migrations/20150618192028-apps-add-lastManifestJson.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastManifestJson VARCHAR(2048)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastManifestJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150710044740-apps-rename-lastBackupConfigJson.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastManifestJson lastBackupConfigJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastBackupConfigJson lastManifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150710170847-apps-add-oldConfigJson.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oldConfigJson VARCHAR(2048)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oldConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150719014338-settings-remove-all.js

@@ -0,0 +1,10 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM settings', [ ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20151013073519-apps-add-oauthProxy.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151015232915-clients-add-type.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'DELETE FROM clients'),
+        db.runSql.bind(db, 'ALTER TABLE clients ADD COLUMN type VARCHAR(16) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE clients DROP COLUMN type', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151016131005-apps-rename-accessRestriction.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestriction accessRestrictionJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestrictionJson accessRestriction VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/initial-schema.sql

@@ -0,0 +1,67 @@
+CREATE TABLE IF NOT EXISTS users(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    username VARCHAR(512) NOT NULL,
+    email VARCHAR(512) NOT NULL,
+    password VARCHAR(1024) NOT NULL,
+    salt VARCHAR(512) NOT NULL,
+    createdAt VARCHAR(512) NOT NULL,
+    modifiedAt VARCHAR(512) NOT NULL,
+    admin INTEGER NOT NULL,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS tokens(
+    accessToken VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128),
+    scope VARCHAR(512) NOT NULL,
+    expires VARCHAR(512) NOT NULL,
+    PRIMARY KEY(accessToken));
+
+CREATE TABLE IF NOT EXISTS clients(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appId VARCHAR(128) NOT NULL,
+    clientSecret VARCHAR(512) NOT NULL,
+    redirectURI VARCHAR(512) NOT NULL,
+    scope VARCHAR(512) NOT NULL,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS apps(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appStoreId VARCHAR(128) NOT NULL,
+    version VARCHAR(32),
+    installationState VARCHAR(512) NOT NULL,
+    installationProgress VARCHAR(512),
+    runState VARCHAR(512),
+    healthy INTEGER,
+    containerId VARCHAR(128),
+    manifestJson VARCHAR(2048),
+    httpPort INTEGER,
+    location VARCHAR(128) NOT NULL UNIQUE,
+    dnsRecordId VARCHAR(512),
+    accessRestriction VARCHAR(512),
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS appPortBindings(
+    hostPort INTEGER NOT NULL UNIQUE,
+    containerPort VARCHAR(5) NOT NULL,
+    appId VARCHAR(128) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id),
+    PRIMARY KEY(hostPort));
+
+CREATE TABLE IF NOT EXISTS authcodes(
+    authCode VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128) NOT NULL,
+    PRIMARY KEY(authCode));
+
+CREATE TABLE IF NOT EXISTS settings(
+    name VARCHAR(128) NOT NULL UNIQUE,
+    value VARCHAR(512),
+    PRIMARY KEY(name));
+
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(128) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id));
+

migrations/schema.sql

@@ -0,0 +1,87 @@
+#### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+#### This file is not used by any code and is here to document the latest schema
+
+#### General ideas
+#### Default char set is utf8 and DEFAULT COLLATE is utf8_bin. Collate affects comparisons in WHERE and ORDER
+#### Strict mode is enabled
+#### VARCHAR - stored as part of table row (use for strings)
+#### TEXT - stored offline from table row (use for strings)
+#### BLOB - stored offline from table row (use for binary data)
+#### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
+
+CREATE TABLE IF NOT EXISTS users(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    email VARCHAR(254) NOT NULL UNIQUE,
+    password VARCHAR(1024) NOT NULL,
+    salt VARCHAR(512) NOT NULL,
+    createdAt VARCHAR(512) NOT NULL,
+    modifiedAt VARCHAR(512) NOT NULL,
+    admin INTEGER NOT NULL,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS tokens(
+    accessToken VARCHAR(128) NOT NULL UNIQUE,
+    identifier VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128),
+    scope VARCHAR(512) NOT NULL,
+    expires BIGINT NOT NULL,
+    PRIMARY KEY(accessToken));
+
+CREATE TABLE IF NOT EXISTS clients(
+    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
+    appId VARCHAR(128) NOT NULL,
+    type VARCHAR(16) NOT NULL,
+    clientSecret VARCHAR(512) NOT NULL,
+    redirectURI VARCHAR(512) NOT NULL,
+    scope VARCHAR(512) NOT NULL,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS apps(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appStoreId VARCHAR(128) NOT NULL,
+    installationState VARCHAR(512) NOT NULL,
+    installationProgress VARCHAR(512),
+    runState VARCHAR(512),
+    health VARCHAR(128),
+    containerId VARCHAR(128),
+    manifestJson VARCHAR(2048),
+    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
+    location VARCHAR(128) NOT NULL UNIQUE,
+    dnsRecordId VARCHAR(512),
+    accessRestrictionJson VARCHAR(2048),
+    oauthProxy BOOLEAN DEFAULT 0,
+    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    lastBackupId VARCHAR(128),
+    lastBackupConfigJson VARCHAR(2048), // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+
+    oldConfigJson VARCHAR(2048), // used to pass old config for apptask
+
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS appPortBindings(
+    hostPort INTEGER NOT NULL UNIQUE,
+    environmentVariable VARCHAR(128) NOT NULL,
+    appId VARCHAR(128) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id),
+    PRIMARY KEY(hostPort));
+
+CREATE TABLE IF NOT EXISTS authcodes(
+    authCode VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128) NOT NULL,
+    expiresAt BIGINT NOT NULL,
+    PRIMARY KEY(authCode));
+
+CREATE TABLE IF NOT EXISTS settings(
+    name VARCHAR(128) NOT NULL UNIQUE,
+    value VARCHAR(512),
+    PRIMARY KEY(name));
+
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(128) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id));
+

package-lock.json

@@ -1,368 +0,0 @@
-{
-  "name": "cloudron-syslog",
-  "version": "1.0.2",
-  "lockfileVersion": 1,
-  "requires": true,
-  "dependencies": {
-    "ansi-regex": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-      "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="
-    },
-    "camelcase": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-      "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0="
-    },
-    "cliui": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz",
-      "integrity": "sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ==",
-      "requires": {
-        "string-width": "2.1.1",
-        "strip-ansi": "4.0.0",
-        "wrap-ansi": "2.1.0"
-      }
-    },
-    "code-point-at": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
-      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
-    },
-    "cross-spawn": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
-      "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
-      "requires": {
-        "lru-cache": "4.1.3",
-        "shebang-command": "1.2.0",
-        "which": "1.3.1"
-      }
-    },
-    "decamelize": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
-      "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
-    },
-    "execa": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
-      "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
-      "requires": {
-        "cross-spawn": "5.1.0",
-        "get-stream": "3.0.0",
-        "is-stream": "1.1.0",
-        "npm-run-path": "2.0.2",
-        "p-finally": "1.0.0",
-        "signal-exit": "3.0.2",
-        "strip-eof": "1.0.0"
-      }
-    },
-    "find-up": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
-      "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
-      "requires": {
-        "locate-path": "2.0.0"
-      }
-    },
-    "get-caller-file": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz",
-      "integrity": "sha1-9wLmMSfn4jHBYKgMFVSstw1QR+U="
-    },
-    "get-stream": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
-      "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ="
-    },
-    "invert-kv": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
-      "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY="
-    },
-    "is-fullwidth-code-point": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-      "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
-    },
-    "is-stream": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
-    },
-    "isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
-    },
-    "lcid": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
-      "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=",
-      "requires": {
-        "invert-kv": "1.0.0"
-      }
-    },
-    "locate-path": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
-      "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
-      "requires": {
-        "p-locate": "2.0.0",
-        "path-exists": "3.0.0"
-      }
-    },
-    "lru-cache": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz",
-      "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==",
-      "requires": {
-        "pseudomap": "1.0.2",
-        "yallist": "2.1.2"
-      }
-    },
-    "mem": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz",
-      "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=",
-      "requires": {
-        "mimic-fn": "1.2.0"
-      }
-    },
-    "mimic-fn": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
-      "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
-    },
-    "minimist": {
-      "version": "0.0.8",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
-    },
-    "mkdirp": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
-      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
-      "requires": {
-        "minimist": "0.0.8"
-      }
-    },
-    "npm-run-path": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
-      "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=",
-      "requires": {
-        "path-key": "2.0.1"
-      }
-    },
-    "nsyslog-parser": {
-      "version": "0.8.1",
-      "resolved": "https://registry.npmjs.org/nsyslog-parser/-/nsyslog-parser-0.8.1.tgz",
-      "integrity": "sha512-4iTO8f0GcdEsuTj1zBCgUiSIIh07Iu6LeUaveDR6yWEHQCU/YJve0xsGZRSbzUVsqafgJ03o8uTfFZh839tyMg=="
-    },
-    "number-is-nan": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
-      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
-    },
-    "os-locale": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz",
-      "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==",
-      "requires": {
-        "execa": "0.7.0",
-        "lcid": "1.0.0",
-        "mem": "1.1.0"
-      }
-    },
-    "p-finally": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
-      "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
-    },
-    "p-limit": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.2.0.tgz",
-      "integrity": "sha512-Y/OtIaXtUPr4/YpMv1pCL5L5ed0rumAaAeBSj12F+bSlMdys7i8oQF/GUJmfpTS/QoaRrS/k6pma29haJpsMng==",
-      "requires": {
-        "p-try": "1.0.0"
-      }
-    },
-    "p-locate": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
-      "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
-      "requires": {
-        "p-limit": "1.2.0"
-      }
-    },
-    "p-try": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
-      "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M="
-    },
-    "path-exists": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-      "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU="
-    },
-    "path-key": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
-      "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
-    },
-    "pseudomap": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
-      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
-    },
-    "require-directory": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
-      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
-    },
-    "require-main-filename": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz",
-      "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE="
-    },
-    "set-blocking": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
-      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
-    },
-    "shebang-command": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
-      "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
-      "requires": {
-        "shebang-regex": "1.0.0"
-      }
-    },
-    "shebang-regex": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
-      "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
-    },
-    "signal-exit": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
-      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
-    },
-    "string-width": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-      "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
-      "requires": {
-        "is-fullwidth-code-point": "2.0.0",
-        "strip-ansi": "4.0.0"
-      }
-    },
-    "strip-ansi": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-      "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
-      "requires": {
-        "ansi-regex": "3.0.0"
-      }
-    },
-    "strip-eof": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
-      "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
-    },
-    "which": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
-      "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
-      "requires": {
-        "isexe": "2.0.0"
-      }
-    },
-    "which-module": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
-      "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
-    },
-    "wrap-ansi": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
-      "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
-      "requires": {
-        "string-width": "1.0.2",
-        "strip-ansi": "3.0.1"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-          "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
-        },
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-          "requires": {
-            "code-point-at": "1.1.0",
-            "is-fullwidth-code-point": "1.0.0",
-            "strip-ansi": "3.0.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
-          "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
-          "requires": {
-            "ansi-regex": "2.1.1"
-          }
-        }
-      }
-    },
-    "y18n": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz",
-      "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE="
-    },
-    "yallist": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
-    },
-    "yargs": {
-      "version": "11.0.0",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.0.0.tgz",
-      "integrity": "sha512-Rjp+lMYQOWtgqojx1dEWorjCofi1YN7AoFvYV7b1gx/7dAAeuI4kN5SZiEvr0ZmsZTOpDRcCqrpI10L31tFkBw==",
-      "requires": {
-        "cliui": "4.1.0",
-        "decamelize": "1.2.0",
-        "find-up": "2.1.0",
-        "get-caller-file": "1.0.2",
-        "os-locale": "2.1.0",
-        "require-directory": "2.1.1",
-        "require-main-filename": "1.0.1",
-        "set-blocking": "2.0.0",
-        "string-width": "2.1.1",
-        "which-module": "2.0.0",
-        "y18n": "3.2.1",
-        "yargs-parser": "9.0.2"
-      }
-    },
-    "yargs-parser": {
-      "version": "9.0.2",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz",
-      "integrity": "sha1-nM9qQ0YP5O1Aqbto9I1DuKaMwHc=",
-      "requires": {
-        "camelcase": "4.1.0"
-      }
-    }
-  }
-}

package.json

@@ -1,23 +1,101 @@
 {
-  "name": "cloudron-syslog",
-  "version": "1.0.2",
-  "description": "Cloudron Syslog Daemon listening on port 2514",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "bin": {
-    "cloudron-syslog": "./index.js"
+  "name": "Cloudron",
+  "description": "Main code for a cloudron",
+  "version": "0.0.1",
+  "private": "true",
+  "author": {
+    "name": "Cloudron authors"
   },
   "repository": {
-    "type": "git",
-    "url": "https://git.cloudron.io/cloudron/cloudron-syslog.git"
+    "type": "git"
   },
-  "author": "Cloudron Developers",
-  "license": "MIT",
+  "engines": [
+    "node >= 0.12.0"
+  ],
   "dependencies": {
-    "mkdirp": "^0.5.1",
-    "nsyslog-parser": "^0.8.1",
-    "yargs": "^11.0.0"
+    "async": "^1.2.1",
+    "aws-sdk": "^2.1.46",
+    "body-parser": "^1.13.1",
+    "bytes": "^2.1.0",
+    "cloudron-manifestformat": "^2.0.0",
+    "connect-ensure-login": "^0.1.1",
+    "connect-lastmile": "0.0.13",
+    "connect-timeout": "^1.5.0",
+    "cookie-parser": "^1.3.5",
+    "cookie-session": "^1.1.0",
+    "cron": "^1.0.9",
+    "csurf": "^1.6.6",
+    "db-migrate": "^0.9.2",
+    "debug": "^2.2.0",
+    "dockerode": "^2.2.2",
+    "ejs": "^2.2.4",
+    "ejs-cli": "^1.0.1",
+    "express": "^4.12.4",
+    "express-session": "^1.11.3",
+    "hat": "0.0.3",
+    "json": "^9.0.3",
+    "ldapjs": "^0.7.1",
+    "memorystream": "^0.3.0",
+    "mime": "^1.3.4",
+    "morgan": "^1.6.0",
+    "multiparty": "^4.1.2",
+    "mysql": "^2.7.0",
+    "native-dns": "^0.7.0",
+    "node-uuid": "^1.4.3",
+    "nodemailer": "^1.3.0",
+    "nodemailer-smtp-transport": "^1.0.3",
+    "oauth2orize": "^1.0.1",
+    "once": "^1.3.2",
+    "passport": "^0.2.2",
+    "passport-http": "^0.2.2",
+    "passport-http-bearer": "^1.0.1",
+    "passport-local": "^1.0.0",
+    "passport-oauth2-client-password": "^0.1.2",
+    "password-generator": "^1.0.0",
+    "proxy-middleware": "^0.13.0",
+    "safetydance": "0.0.19",
+    "semver": "^4.3.6",
+    "serve-favicon": "^2.2.0",
+    "split": "^1.0.0",
+    "superagent": "~0.21.0",
+    "supererror": "^0.7.0",
+    "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "underscore": "^1.7.0",
+    "valid-url": "^1.0.9",
+    "validator": "^3.30.0"
+  },
+  "devDependencies": {
+    "apidoc": "*",
+    "bootstrap-sass": "^3.3.3",
+    "del": "^1.1.1",
+    "expect.js": "*",
+    "gulp": "^3.8.11",
+    "gulp-autoprefixer": "^2.3.0",
+    "gulp-concat": "^2.4.3",
+    "gulp-ejs": "^1.0.0",
+    "gulp-minify-css": "^1.1.3",
+    "gulp-sass": "^2.0.1",
+    "gulp-serve": "^1.0.0",
+    "gulp-sourcemaps": "^1.5.2",
+    "gulp-uglify": "^1.1.0",
+    "hock": "~1.2.0",
+    "istanbul": "*",
+    "js2xmlparser": "^1.0.0",
+    "mocha": "*",
+    "nock": "^2.6.0",
+    "node-sass": "^3.0.0-alpha.0",
+    "redis": "^0.12.1",
+    "request": "^2.65.0",
+    "sinon": "^1.12.2",
+    "yargs": "^3.15.0"
+  },
+  "scripts": {
+    "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
+    "migrate_test": "BOX_ENV=test DATABASE_URL=mysql://root:@localhost/boxtest node_modules/.bin/db-migrate up",
+    "test": "npm run migrate_test && src/test/setupTest && BOX_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
+    "postmerge": "/bin/true",
+    "precommit": "/bin/true",
+    "prepush": "npm test",
+    "webadmin": "node_modules/.bin/gulp"
   }
 }

setup/DESIGN.md

@@ -0,0 +1,57 @@
+This document gives the design of this setup code.
+
+box code should be delivered in the form of a (docker) container.
+This is not the case currently but we want to do structure the code
+in spirit that way.
+
+### container.sh
+This contains code that essential goes into Dockerfile.
+
+This file contains static configuration over a base image. Currently,
+the yellowtent user is created in the installer base image but it
+could very well be placed here.
+
+The idea is that the installer would simply remove the old box container
+and replace it with a new one for an update.
+
+Because we do not package things as Docker yet, we should be careful
+about the code here. We have to expect remains of an older setup code.
+For example, older systemd or nginx configs might be around.
+
+The config directory is _part_ of the container and is not a VOLUME.
+Which is to say that the files will be nuked from one update to the next.
+
+The data directory is a VOLUME. Contents of this directory are expected
+to survive an update. This is a good place to place config files that
+are "dynamic" and need to survive restarts. For example, the infra
+version (see below) or the mysql/postgresql data etc.
+
+### start.sh
+  * It is called in 3 modes - new, update, restore.
+
+  * The first thing this does is to do the static container.sh setup.
+
+  * It then downloads any box restore data and restores the box db from the
+    backup.
+
+  * It then proceeds to call the db-migrate script.
+
+  * It then does dynamic configuration like setting up nginx, collectd.
+
+  * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
+
+  * box services are then started
+
+setup_infra.sh
+This setups containers like graphite, mail and the addons containers.
+
+Containers are relaunched based on the INFRA_VERSION. The script compares
+the version here with the version in the file DATA_DIR/INFRA_VERSION.
+
+If they match, the containers are not recreated and nothing is to be done.
+nginx, collectd configs are part of data already and containers are running.
+
+If they do not match, it deletes all containers (including app containers) and starts
+them all afresh. Important thing here is that, DATA_DIR is never removed across
+updates. So, it is only the containers being recreated and not the data.
+

setup/INFRA_VERSION

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# If you change the infra version, be sure to put a warning
+# in the change log
+
+INFRA_VERSION=19
+
+# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+# These constants are used in the installer script as well
+BASE_IMAGE=cloudron/base:0.7.0
+MYSQL_IMAGE=cloudron/mysql:0.7.0
+POSTGRESQL_IMAGE=cloudron/postgresql:0.7.0
+MONGODB_IMAGE=cloudron/mongodb:0.7.0
+REDIS_IMAGE=cloudron/redis:0.7.0 # if you change this, fix src/addons.js as well
+MAIL_IMAGE=cloudron/mail:0.7.0
+GRAPHITE_IMAGE=cloudron/graphite:0.7.0
+
+MYSQL_REPO=cloudron/mysql
+POSTGRESQL_REPO=cloudron/postgresql
+MONGODB_REPO=cloudron/mongodb
+REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
+MAIL_REPO=cloudron/mail
+GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+json="${script_dir}/../node_modules/.bin/json"
+
+# IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
+arg_api_server_origin=""
+arg_box_versions_url=""
+arg_fqdn=""
+arg_is_custom_domain="false"
+arg_restore_key=""
+arg_restore_url=""
+arg_retire="false"
+arg_tls_cert=""
+arg_tls_key=""
+arg_token=""
+arg_version=""
+arg_web_server_origin=""
+arg_backup_key=""
+arg_aws=""
+
+args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --retire)
+        arg_retire="true"
+        shift
+        ;;
+    --data)
+        # only read mandatory non-empty parameters here
+        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_token arg_is_custom_domain arg_box_versions_url arg_version <<EOF
+        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn token isCustomDomain boxVersionsUrl version | tr '\n' ' ')
+EOF
+        # read possibly empty parameters here
+        arg_tls_cert=$(echo "$2" | $json tlsCert)
+        arg_tls_key=$(echo "$2" | $json tlsKey)
+
+        arg_restore_url=$(echo "$2" | $json restoreUrl)
+        [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
+
+        arg_restore_key=$(echo "$2" | $json restoreKey)
+        [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
+
+        arg_backup_key=$(echo "$2" | $json backupKey)
+        [[ "${arg_backup_key}" == "null" ]] && arg_backup_key=""
+
+        arg_aws=$(echo "$2" | $json aws)
+        [[ "${arg_aws}" == "null" ]] && arg_aws=""
+
+        shift 2
+        ;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+echo "Parsed arguments:"
+echo "api server: ${arg_api_server_origin}"
+echo "box versions url: ${arg_box_versions_url}"
+echo "fqdn: ${arg_fqdn}"
+echo "custom domain: ${arg_is_custom_domain}"
+echo "restore key: ${arg_restore_key}"
+echo "restore url: ${arg_restore_url}"
+echo "tls cert: ${arg_tls_cert}"
+echo "tls key: ${arg_tls_key}"
+echo "token: ${arg_token}"
+echo "version: ${arg_version}"
+echo "web server: ${arg_web_server_origin}"

setup/container.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# This file can be used in Dockerfile
+
+readonly container_files="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/container"
+
+readonly CONFIG_DIR="/home/yellowtent/configs"
+readonly DATA_DIR="/home/yellowtent/data"
+
+########## create config directory
+rm -rf "${CONFIG_DIR}"
+sudo -u yellowtent mkdir "${CONFIG_DIR}"
+
+########## systemd
+rm -f /etc/systemd/system/janitor.*
+cp -r "${container_files}/systemd/." /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable cloudron.target
+
+########## sudoers
+rm -f /etc/sudoers.d/*
+cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
+
+########## collectd
+rm -rf /etc/collectd
+ln -sfF "${DATA_DIR}/collectd" /etc/collectd
+
+########## apparmor docker profile
+cp "${container_files}/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
+systemctl restart apparmor
+
+########## nginx
+# link nginx config to system config
+unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
+ln -s "${DATA_DIR}/nginx" /etc/nginx
+
+########## mysql
+cp "${container_files}/mysql.cnf" /etc/mysql/mysql.cnf
+
+########## Enable services
+update-rc.d -f collectd defaults
+

setup/container/docker-cloudron-app.apparmor

@@ -0,0 +1,32 @@
+#include <tunables/global>
+
+
+profile docker-cloudron-app flags=(attach_disconnected,mediate_deleted) {
+
+  #include <abstractions/base>
+
+  ptrace peer=@{profile_name},
+
+  network,
+  capability,
+  file,
+  umount,
+
+  deny @{PROC}/sys/fs/** wklx,
+  deny @{PROC}/sysrq-trigger rwklx,
+  deny @{PROC}/mem rwklx,
+  deny @{PROC}/kmem rwklx,
+  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
+  deny @{PROC}/sys/kernel/*/** wklx,
+
+  deny mount,
+
+  deny /sys/[^f]*/** wklx,
+  deny /sys/f[^s]*/** wklx,
+  deny /sys/fs/[^c]*/** wklx,
+  deny /sys/fs/c[^g]*/** wklx,
+  deny /sys/fs/cg[^r]*/** wklx,
+  deny /sys/firmware/efi/efivars/** rwklx,
+  deny /sys/kernel/security/** rwklx,
+}
+

setup/container/mysql.cnf

@@ -0,0 +1,7 @@
+!includedir /etc/mysql/conf.d/
+!includedir /etc/mysql/mysql.conf.d/
+
+# http://bugs.mysql.com/bug.php?id=68514
+[mysqld]
+performance_schema=OFF
+max_connection=50

setup/container/sudoers

@@ -0,0 +1,29 @@
+Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupbox.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restoreapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
+
+Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

setup/container/systemd/box.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smart Cloud
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service
+After=box.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/splashpage.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly BOX_SRC_DIR="/home/yellowtent/box"
+readonly DATA_DIR="/home/yellowtent/data"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
+
+echo "Setting up nginx update page"
+
+source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
+
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
+# copy the website
+rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
+cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
+
+# create nginx config
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+    rm -f ${DATA_DIR}/nginx/applications/*
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+else
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+fi
+
+echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+
+nginx -s reload

setup/start.sh

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+echo "==== Cloudron Start ===="
+
+readonly USER="yellowtent"
+readonly BOX_SRC_DIR="/home/${USER}/box"
+readonly DATA_DIR="/home/${USER}/data"
+readonly CONFIG_DIR="/home/${USER}/configs"
+readonly SETUP_PROGRESS_JSON="/home/yellowtent/setup/website/progress.json"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
+
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
+readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
+
+set_progress() {
+    local percent="$1"
+    local message="$2"
+
+    echo "==== ${percent} - ${message} ===="
+    (echo "{ \"update\": { \"percent\": \"${percent}\", \"message\": \"${message}\" }, \"backup\": {} }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
+}
+
+set_progress "1" "Create container"
+$script_dir/container.sh
+
+set_progress "10" "Ensuring directories"
+# keep these in sync with paths.js
+[[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
+mkdir -p "${DATA_DIR}/box/appicons"
+mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/graphite"
+
+mkdir -p "${DATA_DIR}/mysql"
+mkdir -p "${DATA_DIR}/postgresql"
+mkdir -p "${DATA_DIR}/mongodb"
+mkdir -p "${DATA_DIR}/snapshots"
+mkdir -p "${DATA_DIR}/addons"
+mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+
+# bookkeep the version as part of data
+echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
+
+# remove old snapshots. if we do want to keep this around, we will have to fix the chown -R below
+# which currently fails because these are readonly fs
+echo "Cleaning up snapshots"
+find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
+
+# restart mysql to make sure it has latest config
+service mysql restart
+
+readonly mysql_root_password="password"
+mysqladmin -u root -ppassword password password # reset default root password
+mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
+
+if [[ -n "${arg_restore_url}" ]]; then
+    set_progress "15" "Downloading restore data"
+
+    echo "Downloading backup: ${arg_restore_url} and key: ${arg_restore_key}"
+
+    while true; do
+        if $curl -L "${arg_restore_url}" | openssl aes-256-cbc -d -pass "pass:${arg_restore_key}" | tar -zxf - -C "${DATA_DIR}/box"; then break; fi
+        echo "Failed to download data, trying again"
+    done
+
+    set_progress "21" "Setting up MySQL"
+    if [[ -f "${DATA_DIR}/box/box.mysqldump" ]]; then
+        echo "Importing existing database into MySQL"
+        mysql -u root -p${mysql_root_password} box < "${DATA_DIR}/box/box.mysqldump"
+    fi
+fi
+
+set_progress "25" "Migrating data"
+sudo -u "${USER}" -H bash <<EOF
+set -eu
+cd "${BOX_SRC_DIR}"
+BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
+EOF
+
+set_progress "28" "Setup collectd"
+cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
+# collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
+mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
+vda1_id=$(blkid -s UUID -o value /dev/vda1)
+ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
+service collectd restart
+
+set_progress "30" "Setup nginx"
+# setup naked domain to use admin by default. app restoration will overwrite this config
+mkdir -p "${DATA_DIR}/nginx/applications"
+cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"
+
+# generate the main nginx config file
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs" \
+    -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
+
+# generate these for update code paths as well to overwrite splash
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+
+mkdir -p "${DATA_DIR}/nginx/cert"
+echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
+echo "${arg_tls_key}" > ${DATA_DIR}/nginx/cert/host.key
+
+set_progress "33" "Changing ownership"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+chown "${USER}:${USER}" "${DATA_DIR}"
+
+set_progress "40" "Setting up infra"
+${script_dir}/start/setup_infra.sh "${arg_fqdn}"
+
+set_progress "65" "Creating cloudron.conf"
+sudo -u yellowtent -H bash <<EOF
+set -eu
+echo "Creating cloudron.conf"
+# note that arg_aws is a javascript object and intentionally unquoted below
+cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
+{
+    "version": "${arg_version}",
+    "token": "${arg_token}",
+    "apiServerOrigin": "${arg_api_server_origin}",
+    "webServerOrigin": "${arg_web_server_origin}",
+    "fqdn": "${arg_fqdn}",
+    "isCustomDomain": ${arg_is_custom_domain},
+    "boxVersionsUrl": "${arg_box_versions_url}",
+    "adminEmail": "admin@${arg_fqdn}",
+    "database": {
+        "hostname": "localhost",
+        "username": "root",
+        "password": "${mysql_root_password}",
+        "port": 3306,
+        "name": "box"
+    },
+    "backupKey": "${arg_backup_key}",
+    "aws": ${arg_aws}
+}
+CONF_END
+
+echo "Creating config.json for webadmin"
+cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
+{
+    "webServerOrigin": "${arg_web_server_origin}"
+}
+CONF_END
+EOF
+
+# Add webadmin oauth client
+# The domain might have changed, therefor we have to update the record
+# !!! This needs to be in sync with the webadmin, specifically login_callback.js
+echo "Add webadmin oauth cient"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+
+echo "Add localhost test oauth client"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+
+set_progress "80" "Starting Cloudron"
+systemctl start cloudron.target
+
+sleep 2 # give systemd sometime to start the processes
+
+set_progress "85" "Reloading nginx"
+nginx -s reload
+
+set_progress "100" "Done"
+

setup/start/collectd.conf

@@ -0,0 +1,278 @@
+# Config file for collectd(1).
+#
+# Some plugins need additional configuration and are disabled by default.
+# Please read collectd.conf(5) for details.
+#
+# You should also read /usr/share/doc/collectd-core/README.Debian.plugins
+# before enabling any more plugins.
+
+##############################################################################
+# Global                                                                     #
+#----------------------------------------------------------------------------#
+# Global settings for the daemon.                                            #
+##############################################################################
+
+Hostname "localhost"
+#FQDNLookup true
+#BaseDir "/var/lib/collectd"
+#PluginDir "/usr/lib/collectd"
+#TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
+
+#----------------------------------------------------------------------------#
+# When enabled, plugins are loaded automatically with the default options    #
+# when an appropriate <Plugin ...> block is encountered.                     #
+# Disabled by default.                                                       #
+#----------------------------------------------------------------------------#
+#AutoLoadPlugin false
+
+#----------------------------------------------------------------------------#
+# Interval at which to query values. This may be overwritten on a per-plugin #
+# base by using the 'Interval' option of the LoadPlugin block:               #
+#   <LoadPlugin foo>                                                         #
+#       Interval 60                                                          #
+#   </LoadPlugin>                                                            #
+#----------------------------------------------------------------------------#
+# IMPORTANT: changing this value requires a change in whisper schema as well
+Interval 20
+
+#Timeout 2
+#ReadThreads 5
+#WriteThreads 5
+
+# Limit the size of the write queue. Default is no limit. Setting up a limit
+# is recommended for servers handling a high volume of traffic.
+#WriteQueueLimitHigh 1000000
+#WriteQueueLimitLow   800000
+
+##############################################################################
+# Logging                                                                    #
+#----------------------------------------------------------------------------#
+# Plugins which provide logging functions should be loaded first, so log     #
+# messages generated when loading or configuring other plugins can be        #
+# accessed.                                                                  #
+##############################################################################
+
+LoadPlugin logfile
+#LoadPlugin syslog
+
+<Plugin logfile>
+   LogLevel "info"
+   File "/var/log/collectd.log"
+   Timestamp true
+   PrintSeverity false
+</Plugin>
+
+#<Plugin syslog>
+#    LogLevel info
+#</Plugin>
+
+##############################################################################
+# LoadPlugin section                                                         #
+#----------------------------------------------------------------------------#
+# Specify what features to activate.                                         #
+##############################################################################
+
+LoadPlugin aggregation
+#LoadPlugin amqp
+#LoadPlugin apache
+#LoadPlugin apcups
+#LoadPlugin ascent
+#LoadPlugin battery
+#LoadPlugin bind
+#LoadPlugin cgroups
+#LoadPlugin conntrack
+#LoadPlugin contextswitch
+LoadPlugin cpu
+#LoadPlugin cpufreq
+#LoadPlugin csv
+#LoadPlugin curl
+#LoadPlugin curl_json
+#LoadPlugin curl_xml
+#LoadPlugin dbi
+LoadPlugin df
+#LoadPlugin disk
+#LoadPlugin dns
+#LoadPlugin email
+#LoadPlugin entropy
+#LoadPlugin ethstat
+#LoadPlugin exec
+#LoadPlugin filecount
+#LoadPlugin fscache
+#LoadPlugin gmond
+#LoadPlugin hddtemp
+LoadPlugin interface
+#LoadPlugin ipmi
+#LoadPlugin iptables
+#LoadPlugin ipvs
+#LoadPlugin irq
+#LoadPlugin java
+#LoadPlugin libvirt
+LoadPlugin load
+#LoadPlugin lvm
+#LoadPlugin madwifi
+#LoadPlugin mbmon
+#LoadPlugin md
+#LoadPlugin memcachec
+#LoadPlugin memcached
+LoadPlugin memory
+#LoadPlugin modbus
+#LoadPlugin multimeter
+#LoadPlugin mysql
+#LoadPlugin netlink
+#LoadPlugin network
+#LoadPlugin nfs
+LoadPlugin nginx
+#LoadPlugin notify_desktop
+#LoadPlugin notify_email
+#LoadPlugin ntpd
+#LoadPlugin numa
+#LoadPlugin nut
+#LoadPlugin olsrd
+#LoadPlugin openvpn
+#<LoadPlugin perl>
+#   Globals true
+#</LoadPlugin>
+#LoadPlugin pinba
+LoadPlugin ping
+#LoadPlugin postgresql
+#LoadPlugin powerdns
+LoadPlugin processes
+#LoadPlugin protocols
+#<LoadPlugin python>
+#   Globals true
+#</LoadPlugin>
+#LoadPlugin rrdcached
+#LoadPlugin rrdtool
+#LoadPlugin sensors
+#LoadPlugin serial
+#LoadPlugin snmp
+#LoadPlugin statsd
+LoadPlugin swap
+#LoadPlugin table
+LoadPlugin tail
+#LoadPlugin tail_csv
+#LoadPlugin tcpconns
+#LoadPlugin teamspeak2
+#LoadPlugin ted
+#LoadPlugin thermal
+#LoadPlugin tokyotyrant
+#LoadPlugin unixsock
+#LoadPlugin uptime
+#LoadPlugin users
+#LoadPlugin uuid
+#LoadPlugin varnish
+LoadPlugin vmem
+#LoadPlugin vserver
+#LoadPlugin wireless
+LoadPlugin write_graphite
+#LoadPlugin write_http
+#LoadPlugin write_riemann
+
+##############################################################################
+# Plugin configuration                                                       #
+#----------------------------------------------------------------------------#
+# In this section configuration stubs for each plugin are provided. A desc-  #
+# ription of those options is available in the collectd.conf(5) manual page. #
+##############################################################################
+
+<Plugin "aggregation">
+   <Aggregation>
+       Plugin "cpu"
+       Type "cpu"
+
+       GroupBy "Host"
+       GroupBy "TypeInstance"
+
+       CalculateNum false
+       CalculateSum true
+       CalculateAverage true
+       CalculateMinimum false
+       CalculateMaximum true
+       CalculateStddev false
+   </Aggregation>
+</Plugin>
+
+<Plugin df>
+   FSType "tmpfs"
+   MountPoint "/dev"
+
+   ReportByDevice true
+   IgnoreSelected true
+
+   ValuesAbsolute true
+   ValuesPercentage true
+</Plugin>
+
+<Plugin interface>
+   Interface "eth0"
+   IgnoreSelected false
+</Plugin>
+
+<Plugin nginx>
+   URL "http://127.0.0.1/nginx_status"
+</Plugin>
+
+<Plugin ping>
+   Host "google.com"
+   Interval 1.0
+   Timeout 0.9
+   TTL 255
+</Plugin>
+
+<Plugin processes>
+   ProcessMatch "app" "node box.js"
+</Plugin>
+
+<Plugin swap>
+   ReportByDevice false
+   ReportBytes true
+</Plugin>
+
+<Plugin "tail">
+   <File "/var/log/nginx/error.log">
+     Instance "nginx"
+     <Match>
+       Regex ".*"
+       DSType "CounterInc"
+       Type counter
+       Instance "errors"
+     </Match>
+   </File>
+   <File "/var/log/nginx/access.log">
+     Instance "nginx"
+     <Match>
+       Regex ".*"
+       DSType "CounterInc"
+       Type counter
+       Instance "requests"
+     </Match>
+    <Match>
+      Regex " \".*\" [0-9]+ [0-9]+ ([0-9]+)"
+      DSType GaugeAverage
+      Type delay
+      Instance "response"
+    </Match>
+   </File>
+</Plugin>
+
+<Plugin vmem>
+   Verbose false
+</Plugin>
+
+<Plugin write_graphite>
+   <Node "graphing">
+       Host "localhost"
+       Port "2003"
+       Protocol "tcp"
+       LogSendErrors true
+       Prefix "collectd."
+       StoreRates true
+       AlwaysAppendDS false
+       EscapeCharacter "_"
+   </Node>
+</Plugin>
+
+<Include "/etc/collectd/collectd.conf.d">
+    Filter "*.conf"
+</Include>
+

setup/start/nginx/appconfig.ejs

@@ -0,0 +1,113 @@
+# http://nginx.org/en/docs/http/websocket.html
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+server {
+    listen       443;
+    server_name  <%= vhost %>;
+
+    ssl                  on;
+    # paths are relative to prefix and not to this file
+    ssl_certificate      cert/host.cert;
+    ssl_certificate_key  cert/host.key;
+    ssl_session_timeout  5m;
+    ssl_session_cache shared:SSL:50m;
+
+    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
+    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
+    # https://cipherli.st/
+    ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
+
+    proxy_http_version 1.1;
+    proxy_intercept_errors on;
+    proxy_read_timeout       3500;
+    proxy_connect_timeout    3250;
+
+    proxy_set_header   Host               $host;
+    proxy_set_header   X-Forwarded-For    $remote_addr;
+    proxy_set_header   X-Forwarded-Host   $host;
+    proxy_set_header   X-Forwarded-Proto  https;
+
+    # upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    error_page 500 502 503 504 @appstatus;
+    location @appstatus {
+        return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
+    }
+
+    location / {
+        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+        proxy_buffer_size       128k;
+        proxy_buffers           4 256k;
+        proxy_busy_buffers_size 256k;
+
+        # Disable check to allow unlimited body sizes
+        client_max_body_size 0;
+
+<% if ( endpoint === 'admin' ) { %>
+        location /api/ {
+            proxy_pass   http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
+
+        # graphite paths
+        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+            proxy_pass   http://127.0.0.1:8000;
+            client_max_body_size 1m;
+        }
+
+        location / {
+            root   <%= sourceDir %>/webadmin/dist;
+            index  index.html index.htm;
+        }
+
+<% } else if ( endpoint === 'oauthproxy' ) { %>
+        proxy_pass http://127.0.0.1:3003;
+        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
+<% } else if ( endpoint === 'app' ) { %>
+        proxy_pass http://127.0.0.1:<%= port %>;
+<% } else if ( endpoint === 'splash' ) { %>
+        root   <%= sourceDir %>;
+
+        error_page 503 /update.html;
+
+        location /update.html {
+            add_header Cache-Control no-cache;
+        }
+
+        location /theme.css {
+            add_header Cache-Control no-cache;
+        }
+
+        location /3rdparty/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /js/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /progress.json {
+            add_header Cache-Control no-cache;
+        }
+
+        location /api/v1/cloudron/progress {
+            add_header Cache-Control no-cache;
+            default_type application/json;
+            alias <%= sourceDir %>/progress.json;
+        }
+
+        location / {
+            return 503;
+        }
+<% } %>
+    }
+}
+

setup/start/nginx/mime.types

@@ -0,0 +1,80 @@
+
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/x-javascript              js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg svgz;
+    image/webp                            webp;
+
+    application/java-archive              jar war ear;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/xhtml+xml                 xhtml;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              eot;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/ogg                             ogg;
+    audio/x-m4a                           m4a;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mp4                             mp4;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/webm                            webm;
+    video/x-flv                           flv;
+    video/x-m4v                           m4v;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

setup/start/nginx/nginx.ejs

@@ -0,0 +1,67 @@
+user www-data;
+
+worker_processes  1;
+
+pid /run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    # the collectd config depends on this log format
+    log_format combined2 '$remote_addr - [$time_local] '
+        '"$request" $status $body_bytes_sent $request_time '
+        '"$http_referer" "$http_user_agent"';
+
+    # required for long host names
+    server_names_hash_bucket_size 128;
+
+    access_log access.log combined2;
+
+    sendfile        on;
+
+    keepalive_timeout  65;
+
+    # HTTP server
+    server {
+        listen      80;
+
+        # collectd
+        location /nginx_status {
+            stub_status on;
+            access_log off;
+            allow 127.0.0.1;
+            deny all;
+        }
+
+        location / {
+            # redirect everything to HTTPS
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    # We have to enable https for nginx to read in the vhost in http request
+    # and send a 404. This is a side-effect of using wildcard DNS
+    server {
+        listen 443 default_server;
+        ssl on;
+        ssl_certificate      cert/host.cert;
+        ssl_certificate_key  cert/host.key;
+
+        error_page 404 = @fallback;
+        location @fallback {
+            internal;
+            root <%= sourceDir %>/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
+        return 404;
+    }
+
+    include applications/*.conf;
+}
+

setup/start/setup_infra.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly DATA_DIR="/home/yellowtent/data"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
+
+arg_fqdn="$1"
+
+# removing containers ensures containers are launched with latest config updates
+# restore code in appatask does not delete old containers
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
+    echo "Infrastructure is upto date"
+    exit 0
+fi
+
+echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
+
+existing_containers=$(docker ps -qa)
+echo "Remove containers: ${existing_containers}"
+if [[ -n "${existing_containers}" ]]; then
+    echo "${existing_containers}" | xargs docker rm -f
+fi
+
+# graphite
+graphite_container_id=$(docker run --restart=always -d --name="graphite" \
+    -m 75m \
+    --memory-swap 150m \
+    -p 127.0.0.1:2003:2003 \
+    -p 127.0.0.1:2004:2004 \
+    -p 127.0.0.1:8000:8000 \
+    -v "${DATA_DIR}/graphite:/app/data" \
+    --read-only -v /tmp -v /run \
+    "${GRAPHITE_IMAGE}")
+echo "Graphite container id: ${graphite_container_id}"
+if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old graphite images"
+fi
+
+# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
+mail_container_id=$(docker run --restart=always -d --name="mail" \
+    -m 75m \
+    --memory-swap 150m \
+    -h "${arg_fqdn}" \
+    -e "DOMAIN_NAME=${arg_fqdn}" \
+    -v "${DATA_DIR}/box/mail:/app/data" \
+    --read-only -v /tmp -v /run \
+    "${MAIL_IMAGE}")
+echo "Mail container id: ${mail_container_id}"
+if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mail images"
+fi
+
+# mysql
+mysql_addon_root_password=$(pwgen -1 -s)
+docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
+cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
+readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
+readonly MYSQL_ROOT_HOST='${docker0_ip}'
+EOF
+mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -m 100m \
+    --memory-swap 200m \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
+    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
+    "${MYSQL_IMAGE}")
+echo "MySQL container id: ${mysql_container_id}"
+if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mysql images"
+fi
+
+# postgresql
+postgresql_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
+readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
+EOF
+postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -m 100m \
+    --memory-swap 200m \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
+    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
+    "${POSTGRESQL_IMAGE}")
+echo "PostgreSQL container id: ${postgresql_container_id}"
+if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old postgresql images"
+fi
+
+# mongodb
+mongodb_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
+readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
+EOF
+mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -m 100m \
+    --memory-swap 200m \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
+    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
+    "${MONGODB_IMAGE}")
+echo "Mongodb container id: ${mongodb_container_id}"
+if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mongodb images"
+fi
+
+# redis
+if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old redis images"
+fi
+
+# only touch apps in installed state. any other state is just resumed by the taskmanager
+if [[ "${infra_version}" == "none" ]]; then
+    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
+    echo "Marking installed apps for restore"
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
+else
+    # if existing infra was found, just mark apps for reconfiguration
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
+fi
+
+echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"

setup/stop.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+echo "Stopping cloudron"
+
+systemctl stop cloudron.target

src/addons.js

@@ -0,0 +1,848 @@
+'use strict';
+
+exports = module.exports = {
+    setupAddons: setupAddons,
+    teardownAddons: teardownAddons,
+    backupAddons: backupAddons,
+    restoreAddons: restoreAddons,
+
+    getEnvironment: getEnvironment,
+    getLinksSync: getLinksSync,
+    getBindsSync: getBindsSync,
+
+    // exported for testing
+    _setupOauth: setupOauth,
+    _teardownOauth: teardownOauth
+};
+
+var appdb = require('./appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    child_process = require('child_process'),
+    clientdb = require('./clientdb.js'),
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:addons'),
+    docker = require('./docker.js').connection,
+    fs = require('fs'),
+    generatePassword = require('password-generator'),
+    hat = require('hat'),
+    MemoryStream = require('memorystream'),
+    once = require('once'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    spawn = child_process.spawn,
+    util = require('util'),
+    uuid = require('node-uuid');
+
+var NOOP = function (app, options, callback) { return callback(); };
+
+// setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
+// teardown is destructive. app data stored with the addon is lost
+var KNOWN_ADDONS = {
+    ldap: {
+        setup: setupLdap,
+        teardown: teardownLdap,
+        backup: NOOP,
+        restore: setupLdap
+    },
+    localstorage: {
+        setup: NOOP, // docker creates the directory for us
+        teardown: NOOP,
+        backup: NOOP, // no backup because it's already inside app data
+        restore: NOOP
+    },
+    mongodb: {
+        setup: setupMongoDb,
+        teardown: teardownMongoDb,
+        backup: backupMongoDb,
+        restore: restoreMongoDb
+    },
+    mysql: {
+        setup: setupMySql,
+        teardown: teardownMySql,
+        backup: backupMySql,
+        restore: restoreMySql,
+    },
+    oauth: {
+        setup: setupOauth,
+        teardown: teardownOauth,
+        backup: NOOP,
+        restore: setupOauth
+    },
+    postgresql: {
+        setup: setupPostgreSql,
+        teardown: teardownPostgreSql,
+        backup: backupPostgreSql,
+        restore: restorePostgreSql
+    },
+    redis: {
+        setup: setupRedis,
+        teardown: teardownRedis,
+        backup: backupRedis,
+        restore: setupRedis // same thing
+    },
+    sendmail: {
+        setup: setupSendMail,
+        teardown: teardownSendMail,
+        backup: NOOP,
+        restore: setupSendMail
+    },
+    scheduler: {
+        setup: NOOP,
+        teardown: NOOP,
+        backup: NOOP,
+        restore: NOOP
+    },
+    simpleauth: {
+        setup: setupSimpleAuth,
+        teardown: teardownSimpleAuth,
+        backup: NOOP,
+        restore: setupSimpleAuth
+    },
+    _docker: {
+        setup: NOOP,
+        teardown: NOOP,
+        backup: NOOP,
+        restore: NOOP
+    }
+};
+
+var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh');
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function setupAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!addons) return callback(null);
+
+    debugApp(app, 'setupAddons: Settings up %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+
+        debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);
+
+        KNOWN_ADDONS[addon].setup(app, addons[addon], iteratorCallback);
+    }, callback);
+}
+
+function teardownAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!addons) return callback(null);
+
+    debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+
+        debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);
+
+        KNOWN_ADDONS[addon].teardown(app, addons[addon], iteratorCallback);
+    }, callback);
+}
+
+function backupAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'backupAddons');
+
+    if (!addons) return callback(null);
+
+    debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+
+        KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
+    }, callback);
+}
+
+function restoreAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'restoreAddons');
+
+    if (!addons) return callback(null);
+
+    debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+
+        KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
+    }, callback);
+}
+
+function getEnvironment(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAddonConfigByAppId(app.id, callback);
+}
+
+function getLinksSync(app, addons) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+
+    var links = [ ];
+
+    if (!addons) return links;
+
+    for (var addon in addons) {
+        switch (addon) {
+        case 'mysql': links.push('mysql:mysql'); break;
+        case 'postgresql': links.push('postgresql:postgresql'); break;
+        case 'sendmail': links.push('mail:mail'); break;
+        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
+        case 'mongodb': links.push('mongodb:mongodb'); break;
+        default: break;
+        }
+    }
+
+    return links;
+}
+
+function getBindsSync(app, addons) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+
+    var binds = [ ];
+
+    if (!addons) return binds;
+
+    for (var addon in addons) {
+        switch (addon) {
+        case '_docker': binds.push('/var/run/docker.sock:/var/run/docker.sock:rw'); break;
+        case 'localstorage': binds.push(path.join(paths.DATA_DIR, app.id, 'data') + ':/app/data:rw'); break;
+        default: break;
+        }
+    }
+
+    return binds;
+}
+
+function setupOauth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appId = app.id;
+    var id = 'cid-' + uuid.v4();
+    var clientSecret = hat(256);
+    var redirectURI = 'https://' + config.appFqdn(app.location);
+    var scope = 'profile';
+
+    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
+
+    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
+        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
+
+        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+            if (error) return callback(error);
+
+            var env = [
+                'OAUTH_CLIENT_ID=' + id,
+                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_ORIGIN=' + config.adminOrigin()
+            ];
+
+            debugApp(app, 'Setting oauth addon config to %j', env);
+
+            appdb.setAddonConfig(appId, 'oauth', env, callback);
+        });
+    });
+}
+
+function teardownOauth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'teardownOauth');
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+
+        appdb.unsetAddonConfig(app.id, 'oauth', callback);
+    });
+}
+
+function setupSimpleAuth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appId = app.id;
+    var id = 'cid-' + uuid.v4();
+    var scope = 'profile';
+
+    debugApp(app, 'setupSimpleAuth: id:%s', id);
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
+
+        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+            if (error) return callback(error);
+
+            var env = [
+                'SIMPLE_AUTH_SERVER=172.17.42.1',
+                'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_URL=http://172.17.42.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.17.42.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + id
+            ];
+
+            debugApp(app, 'Setting simple auth addon config to %j', env);
+
+            appdb.setAddonConfig(appId, 'simpleauth', env, callback);
+        });
+    });
+}
+
+function teardownSimpleAuth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'teardownSimpleAuth');
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+
+        appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
+    });
+}
+
+function setupLdap(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var env = [
+        'LDAP_SERVER=172.17.42.1',
+        'LDAP_PORT=' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.17.42.1:' + config.get('ldapPort'),
+        'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
+        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
+        'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
+        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
+    ];
+
+    debugApp(app, 'Setting up LDAP');
+
+    appdb.setAddonConfig(app.id, 'ldap', env, callback);
+}
+
+function teardownLdap(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down LDAP');
+
+    appdb.unsetAddonConfig(app.id, 'ldap', callback);
+}
+
+function setupSendMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
+        'MAIL_SMTP_USERNAME=' + (app.location || app.id), // use app.id for bare domains
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up sendmail');
+
+    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+}
+
+function teardownSendMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down sendmail');
+
+    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+}
+
+function setupMySql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up mysql');
+
+    var container = docker.getContainer('mysql');
+    var cmd = [ '/addons/mysql/service.sh', 'add', app.id ];
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var stdout = new MemoryStream();
+            var stderr = new MemoryStream();
+
+            execContainer.modem.demuxStream(stream, stdout, stderr);
+            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
+
+            var chunks = [ ];
+            stdout.on('data', function (chunk) { chunks.push(chunk); });
+
+            stream.on('error', callback);
+            stream.on('end', function () {
+                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+                debugApp(app, 'Setting mysql addon config to %j', env);
+                appdb.setAddonConfig(app.id, 'mysql', env, callback);
+            });
+        });
+    });
+}
+
+function teardownMySql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = docker.getContainer('mysql');
+    var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
+
+    debugApp(app, 'Tearing down mysql');
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var data = '';
+            stream.on('error', callback);
+            stream.on('data', function (d) { data += d.toString('utf8'); });
+            stream.on('end', function () {
+                appdb.unsetAddonConfig(app.id, 'mysql', callback);
+            });
+        });
+    });
+}
+
+function backupMySql(app, options, callback) {
+    debugApp(app, 'Backing up mysql');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
+    output.on('error', callback);
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', 'backup', app.id ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
+        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(output);
+    cp.stderr.pipe(process.stderr);
+}
+
+function restoreMySql(app, options, callback) {
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    setupMySql(app, options, function (error) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreMySql');
+
+        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
+        input.on('error', callback);
+
+        // cannot get this to work through docker.exec
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', 'restore', app.id ]);
+        cp.on('error', callback);
+        cp.on('exit', function (code, signal) {
+            debugApp(app, 'restoreMySql: done %s %s', code, signal);
+            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
+        });
+
+        cp.stdout.pipe(process.stdout);
+        cp.stderr.pipe(process.stderr);
+        input.pipe(cp.stdin).on('error', callback);
+    });
+}
+
+function setupPostgreSql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up postgresql');
+
+    var container = docker.getContainer('postgresql');
+    var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var stdout = new MemoryStream();
+            var stderr = new MemoryStream();
+
+            execContainer.modem.demuxStream(stream, stdout, stderr);
+            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
+
+            var chunks = [ ];
+            stdout.on('data', function (chunk) { chunks.push(chunk); });
+
+            stream.on('error', callback);
+            stream.on('end', function () {
+                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+                debugApp(app, 'Setting postgresql addon config to %j', env);
+                appdb.setAddonConfig(app.id, 'postgresql', env, callback);
+            });
+        });
+    });
+}
+
+function teardownPostgreSql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = docker.getContainer('postgresql');
+    var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
+
+    debugApp(app, 'Tearing down postgresql');
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var data = '';
+            stream.on('error', callback);
+            stream.on('data', function (d) { data += d.toString('utf8'); });
+            stream.on('end', function () {
+                appdb.unsetAddonConfig(app.id, 'postgresql', callback);
+            });
+        });
+    });
+}
+
+function backupPostgreSql(app, options, callback) {
+    debugApp(app, 'Backing up postgresql');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
+    output.on('error', callback);
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
+        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(output);
+    cp.stderr.pipe(process.stderr);
+}
+
+function restorePostgreSql(app, options, callback) {
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    setupPostgreSql(app, options, function (error) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restorePostgreSql');
+
+        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
+        input.on('error', callback);
+
+        // cannot get this to work through docker.exec
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
+        cp.on('error', callback);
+        cp.on('exit', function (code, signal) {
+            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
+            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
+        });
+
+        cp.stdout.pipe(process.stdout);
+        cp.stderr.pipe(process.stderr);
+        input.pipe(cp.stdin).on('error', callback);
+    });
+}
+
+function setupMongoDb(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up mongodb');
+
+    var container = docker.getContainer('mongodb');
+    var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var stdout = new MemoryStream();
+            var stderr = new MemoryStream();
+
+            execContainer.modem.demuxStream(stream, stdout, stderr);
+            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
+
+            var chunks = [ ];
+            stdout.on('data', function (chunk) { chunks.push(chunk); });
+
+            stream.on('error', callback);
+            stream.on('end', function () {
+                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+                debugApp(app, 'Setting mongodb addon config to %j', env);
+                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
+            });
+        });
+    });
+}
+
+function teardownMongoDb(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = docker.getContainer('mongodb');
+    var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
+
+    debugApp(app, 'Tearing down mongodb');
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var data = '';
+            stream.on('error', callback);
+            stream.on('data', function (d) { data += d.toString('utf8'); });
+            stream.on('end', function () {
+                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
+            });
+        });
+    });
+}
+
+function backupMongoDb(app, options, callback) {
+    debugApp(app, 'Backing up mongodb');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
+    output.on('error', callback);
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
+        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(output);
+    cp.stderr.pipe(process.stderr);
+}
+
+function restoreMongoDb(app, options, callback) {
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    setupMongoDb(app, options, function (error) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreMongoDb');
+
+        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
+        input.on('error', callback);
+
+        // cannot get this to work through docker.exec
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
+        cp.on('error', callback);
+        cp.on('exit', function (code, signal) {
+            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
+            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
+        });
+
+        cp.stdout.pipe(process.stdout);
+        cp.stderr.pipe(process.stderr);
+        input.pipe(cp.stdin).on('error', callback);
+    });
+}
+
+
+function forwardRedisPort(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.getContainer('redis-' + appId).inspect(function (error, data) {
+        if (error) return callback(new Error('Unable to inspect container:' + error));
+
+        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
+        if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
+
+        return callback(null);
+    });
+}
+
+function stopAndRemoveRedis(container, callback) {
+    function ignoreError(func) {
+        return function (callback) {
+            func(function (error) {
+                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
+                callback();
+            });
+        };
+    }
+
+    // stopping redis with SIGTERM makes it commit the database to disk
+    async.series([
+        ignoreError(container.stop.bind(container, { t: 10 })),
+        ignoreError(container.wait.bind(container)),
+        ignoreError(container.remove.bind(container, { force: true, v: true }))
+    ], callback);
+}
+
+// Ensures that app's addon redis container is running. Can be called when named container already exists/running
+function setupRedis(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var redisPassword = generatePassword(64, false /* memorable */);
+    var redisVarsFile = path.join(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
+    var redisDataDir = path.join(paths.DATA_DIR, app.id + '/redis');
+
+    if (!safe.fs.writeFileSync(redisVarsFile, 'REDIS_PASSWORD=' + redisPassword)) {
+        return callback(new Error('Error writing redis config'));
+    }
+
+    if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error));
+
+    var createOptions = {
+        name: 'redis-' + app.id,
+        Hostname: 'redis-' + app.location,
+        Tty: true,
+        Image: 'cloudron/redis:0.7.0', // if you change this, fix setup/INFRA_VERSION as well
+        Cmd: null,
+        Volumes: {
+            '/tmp': {},
+            '/run': {}
+        },
+        VolumesFrom: [],
+        HostConfig: {
+            Binds: [
+                redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
+                redisDataDir + ':/var/lib/redis:rw'
+            ],
+            Memory: 1024 * 1024 * 75, // 100mb
+            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
+            PortBindings: {
+                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
+            },
+            ReadonlyRootfs: true,
+            RestartPolicy: {
+                'Name': 'always',
+                'MaximumRetryCount': 0
+            }
+        }
+    };
+
+    var env = [
+        'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id,
+        'REDIS_PASSWORD=' + redisPassword,
+        'REDIS_HOST=redis-' + app.id,
+        'REDIS_PORT=6379'
+    ];
+
+    var redisContainer = docker.getContainer(createOptions.name);
+    stopAndRemoveRedis(redisContainer, function () {
+        docker.createContainer(createOptions, function (error) {
+            if (error && error.statusCode !== 409) return callback(error); // if not already created
+
+            redisContainer.start(function (error) {
+                if (error && error.statusCode !== 304) return callback(error); // if not already running
+
+                appdb.setAddonConfig(app.id, 'redis', env, function (error) {
+                    if (error) return callback(error);
+
+                    forwardRedisPort(app.id, callback);
+                });
+            });
+        });
+    });
+}
+
+function teardownRedis(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+   var container = docker.getContainer('redis-' + app.id);
+
+   var removeOptions = {
+       force: true, // kill container if it's running
+       v: true // removes volumes associated with the container
+   };
+
+   container.remove(removeOptions, function (error) {
+       if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
+
+       safe.fs.unlinkSync(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
+
+        shell.sudo('teardownRedis', [ RMAPPDIR_CMD, app.id + '/redis' ], function (error, stdout, stderr) {
+            if (error) return callback(new Error('Error removing redis data:' + error));
+
+            appdb.unsetAddonConfig(app.id, 'redis', callback);
+        });
+   });
+}
+
+function backupRedis(app, options, callback) {
+    debugApp(app, 'Backing up redis');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
+        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(process.stdout);
+    cp.stderr.pipe(process.stderr);
+}

src/aes-helper.js

@@ -0,0 +1,49 @@
+'use strict';
+
+var crypto = require('crypto');
+
+// This code is taken from https://github.com/fabdrol/node-aes-helper
+module.exports = {
+    algorithm: 'AES-256-CBC',
+
+    key: function (password, salt) {
+        var key = salt.toString('utf8') + password;
+        var hash = crypto.createHash('sha1');
+
+        hash.update(key, 'utf8');
+        return hash.digest('hex');
+    },
+
+    encrypt: function (plain, password, salt) {
+        var key = this.key(password, salt);
+        var cipher = crypto.createCipher(this.algorithm, key);
+        var crypted;
+
+        try {
+            crypted = cipher.update(plain, 'utf8', 'hex');
+            crypted += cipher.final('hex');
+        } catch (e) {
+            console.error('Encryption error:', e);
+            crypted = '';
+        }
+
+        return crypted;
+    },
+
+    decrypt: function (crypted, password, salt) {
+        var key = this.key(password, salt);
+        var decipher = crypto.createDecipher(this.algorithm, key);
+        var decoded;
+
+        try {
+            decoded = decipher.update(crypted, 'hex', 'utf8');
+            decoded += decipher.final('utf8');
+        } catch (e) {
+            console.error('Decryption error:', e);
+            decoded = '';
+        }
+
+        return decoded;
+    }
+};
+

src/appdb.js

@@ -0,0 +1,474 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getBySubdomain: getBySubdomain,
+    getByHttpPort: getByHttpPort,
+    getByContainerId: getByContainerId,
+    add: add,
+    exists: exists,
+    del: del,
+    update: update,
+    getAll: getAll,
+    getPortBindings: getPortBindings,
+
+    setAddonConfig: setAddonConfig,
+    getAddonConfig: getAddonConfig,
+    getAddonConfigByAppId: getAddonConfigByAppId,
+    unsetAddonConfig: unsetAddonConfig,
+    unsetAddonConfigByAppId: unsetAddonConfigByAppId,
+
+    setHealth: setHealth,
+    setInstallationCommand: setInstallationCommand,
+    setRunCommand: setRunCommand,
+    getAppStoreIds: getAppStoreIds,
+
+    // installation codes (keep in sync in UI)
+    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
+    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
+    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
+    ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
+    ISTATE_PENDING_FORCE_UPDATE: 'pending_force_update', // update from any state preserving data
+    ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app
+    ISTATE_ERROR: 'error', // error executing last pending_* command
+    ISTATE_INSTALLED: 'installed', // app is installed
+
+    RSTATE_RUNNING: 'running',
+    RSTATE_PENDING_START: 'pending_start',
+    RSTATE_PENDING_STOP: 'pending_stop',
+    RSTATE_STOPPED: 'stopped', // app stopped by use
+
+    // run codes (keep in sync in UI)
+    HEALTH_HEALTHY: 'healthy',
+    HEALTH_UNHEALTHY: 'unhealthy',
+    HEALTH_ERROR: 'error',
+    HEALTH_DEAD: 'dead',
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    safe = require('safetydance'),
+    util = require('util');
+
+var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
+
+var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
+    result.manifest = safe.JSON.parse(result.manifestJson);
+    delete result.manifestJson;
+
+    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
+    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
+    delete result.lastBackupConfigJson;
+
+    assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
+    result.oldConfig = safe.JSON.parse(result.oldConfigJson);
+    delete result.oldConfigJson;
+
+    assert(result.hostPorts === null || typeof result.hostPorts === 'string');
+    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
+
+    result.portBindings = { };
+    var hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
+    var environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
+
+    delete result.hostPorts;
+    delete result.environmentVariables;
+
+    for (var i = 0; i < environmentVariables.length; i++) {
+        result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
+    }
+
+    result.oauthProxy = !!result.oauthProxy;
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getBySubdomain(subdomain, callback) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getByHttpPort(httpPort, callback) {
+    assert.strictEqual(typeof httpPort, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getByContainerId(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
+        + ' GROUP BY apps.id ORDER BY apps.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof manifest.version, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    portBindings = portBindings || { };
+
+    var manifestJson = JSON.stringify(manifest);
+    var accessRestrictionJson = JSON.stringify(accessRestriction);
+
+    var queries = [ ];
+    queries.push({
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
+    });
+
+    Object.keys(portBindings).forEach(function (env) {
+        queries.push({
+            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, appId) VALUES (?, ?, ?)',
+            args: [ env, portBindings[env], id ]
+        });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function exists(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result.length !== 0);
+    });
+}
+
+function getPortBindings(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        var portBindings = { };
+        for (var i = 0; i < results.length; i++) {
+            portBindings[results[i].environmentVariable] = results[i].hostPort;
+        }
+
+        callback(null, portBindings);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [
+        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
+    ];
+
+    database.transaction(queries, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        database.query.bind(null, 'DELETE FROM appPortBindings'),
+        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
+        database.query.bind(null, 'DELETE FROM apps')
+    ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        return callback(null);
+    });
+}
+
+function update(id, app, callback) {
+    updateWithConstraints(id, app, '', callback);
+}
+
+function updateWithConstraints(id, app, constraints, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof constraints, 'string');
+    assert.strictEqual(typeof callback, 'function');
+    assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+
+    var queries = [ ];
+
+    if ('portBindings' in app) {
+        var portBindings = app.portBindings || { };
+        // replace entries by app id
+        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
+        Object.keys(portBindings).forEach(function (env) {
+            var values = [ portBindings[env], env, id ];
+            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, environmentVariable, appId) VALUES(?, ?, ?)', args: values });
+        });
+    }
+
+    var fields = [ ], values = [ ];
+    for (var p in app) {
+        if (p === 'manifest') {
+            fields.push('manifestJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'lastBackupConfig') {
+            fields.push('lastBackupConfigJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'oldConfig') {
+            fields.push('oldConfigJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'accessRestriction') {
+            fields.push('accessRestrictionJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p !== 'portBindings') {
+            fields.push(p + ' = ?');
+            values.push(app[p]);
+        }
+    }
+
+    if (values.length !== 0) {
+        values.push(id);
+        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
+    }
+
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results[results.length - 1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
+// not sure if health should influence runState
+function setHealth(appId, health, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var values = { health: health };
+
+    var constraints = 'AND runState NOT LIKE "pending_%" AND installationState = "installed"';
+
+    updateWithConstraints(appId, values, constraints, callback);
+}
+
+function setInstallationCommand(appId, installationState, values, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof installationState, 'string');
+
+    if (typeof values === 'function') {
+        callback = values;
+        values = { };
+    } else {
+        assert.strictEqual(typeof values, 'object');
+        assert.strictEqual(typeof callback, 'function');
+    }
+
+    values.installationState = installationState;
+    values.installationProgress = '';
+
+    // Rules are:
+    // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
+    // restore is allowed from installed or error state
+    // update and configure are allowed only in installed state
+
+    if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
+        updateWithConstraints(appId, values, '', callback);
+    } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
+        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
+        updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else {
+        callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
+    }
+}
+
+function setRunCommand(appId, runState, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof runState, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var values = { runState: runState };
+    updateWithConstraints(appId, values, 'AND runState NOT LIKE "pending_%" AND installationState = "installed"', callback);
+}
+
+function getAppStoreIds(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAddonConfig(appId, addonId, env, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(util.isArray(env));
+    assert.strictEqual(typeof callback, 'function');
+
+    unsetAddonConfig(appId, addonId, function (error) {
+        if (error) return callback(error);
+
+        if (env.length === 0) return callback(null);
+
+        var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
+        var args = [ ], queryArgs = [ ];
+        for (var i = 0; i < env.length; i++) {
+            args.push(appId, addonId, env[i]);
+            queryArgs.push('(?, ?, ?)');
+        }
+
+        database.query(query + queryArgs.join(','), args, function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            return callback(null);
+        });
+    });
+}
+
+function unsetAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function unsetAddonConfigByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        var config = [ ];
+        results.forEach(function (v) { config.push(v.value); });
+
+        callback(null, config);
+    });
+}
+
+function getAddonConfigByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        var config = [ ];
+        results.forEach(function (v) { config.push(v.value); });
+
+        callback(null, config);
+    });
+}
+

src/apphealthmonitor.js

@@ -0,0 +1,187 @@
+'use strict';
+
+var appdb = require('./appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apphealthmonitor'),
+    docker = require('./docker.js').connection,
+    mailer = require('./mailer.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
+var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var gHealthInfo = { }; // { time, emailSent }
+var gRunTimeout = null;
+var gDockerEventStream = null;
+
+function debugApp(app) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function setHealth(app, health, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+
+    if (!(app.id in gHealthInfo)) { // add new apps to list
+        gHealthInfo[app.id] = { time: now, emailSent: false };
+    }
+
+    if (health === appdb.HEALTH_HEALTHY) {
+        gHealthInfo[app.id].time = now;
+    } else if (Math.abs(now - gHealthInfo[app.id].time) > UNHEALTHY_THRESHOLD) {
+        if (gHealthInfo[app.id].emailSent) return callback(null);
+
+        debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
+
+        if (app.appStoreId !== '') mailer.appDied(app); // do not send mails for dev apps
+        gHealthInfo[app.id].emailSent = true;
+    } else {
+        debugApp(app, 'waiting for sometime to update the app health');
+        return callback(null);
+    }
+
+    appdb.setHealth(app.id, health, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled?
+        if (error) return callback(error);
+
+        app.health = health;
+
+        callback(null);
+    });
+}
+
+
+// callback is called with error for fatal errors and not if health check failed
+function checkAppHealth(app, callback) {
+    if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
+        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
+        return callback(null);
+    }
+
+    var container = docker.getContainer(app.containerId),
+        manifest = app.manifest;
+
+    container.inspect(function (err, data) {
+        if (err || !data || !data.State) {
+            debugApp(app, 'Error inspecting container');
+            return setHealth(app, appdb.HEALTH_ERROR, callback);
+        }
+
+        if (data.State.Running !== true) {
+            debugApp(app, 'exited');
+            return setHealth(app, appdb.HEALTH_DEAD, callback);
+        }
+
+        // poll through docker network instead of nginx to bypass any potential oauth proxy
+        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
+        superagent
+            .get(healthCheckUrl)
+            .redirects(0)
+            .timeout(HEALTHCHECK_INTERVAL)
+            .end(function (error, res) {
+
+            if (error || res.status >= 400) { // 2xx and 3xx are ok
+                debugApp(app, 'not alive : %s', error || res.status);
+                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
+            } else {
+                debugApp(app, 'alive');
+                setHealth(app, appdb.HEALTH_HEALTHY, callback);
+            }
+        });
+    });
+}
+
+function processApps(callback) {
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(error);
+
+        async.each(apps, checkAppHealth, function (error) {
+            if (error) console.error(error);
+            callback(null);
+        });
+    });
+}
+
+function run() {
+    processApps(function (error) {
+        if (error) console.error(error);
+
+        gRunTimeout = setTimeout(run, HEALTHCHECK_INTERVAL);
+    });
+}
+
+/*
+    OOM can be tested using stress tool like so:
+        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        apt-get update && apt-get install stress
+        stress --vm 1 --vm-bytes 200M --vm-hang 0
+*/
+function processDockerEvents() {
+    // note that for some reason, the callback is called only on the first event
+    debug('Listening for docker events');
+    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return console.error(error);
+
+        gDockerEventStream = stream;
+
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            var ev = JSON.parse(data);
+            debug('Container ' + ev.id + ' went OOM');
+            appdb.getByContainerId(ev.id, function (error, app) {
+                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
+                var context = JSON.stringify(ev);
+                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
+
+                debug('OOM Context: %s', context);
+
+                // do not send mails for dev apps
+                if (app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+            });
+        });
+
+        stream.on('error', function (error) {
+            console.error('Error reading docker events', error);
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+
+        stream.on('end', function () {
+            console.error('Docker event stream ended');
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+    });
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting apphealthmonitor');
+
+    processDockerEvents();
+
+    run();
+
+    callback();
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    clearTimeout(gRunTimeout);
+    gDockerEventStream.end();
+
+    callback();
+}

src/apps.js

@@ -0,0 +1,846 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    AppsError: AppsError,
+
+    hasAccessTo: hasAccessTo,
+
+    get: get,
+    getBySubdomain: getBySubdomain,
+    getAll: getAll,
+    purchase: purchase,
+    install: install,
+    configure: configure,
+    uninstall: uninstall,
+
+    restore: restore,
+    restoreApp: restoreApp,
+
+    update: update,
+
+    backup: backup,
+    backupApp: backupApp,
+
+    getLogStream: getLogStream,
+    getLogs: getLogs,
+
+    start: start,
+    stop: stop,
+
+    exec: exec,
+
+    checkManifestConstraints: checkManifestConstraints,
+
+    setRestorePoint: setRestorePoint,
+
+    autoupdateApps: autoupdateApps,
+
+    // exported for testing
+    _validateHostname: validateHostname,
+    _validatePortBindings: validatePortBindings,
+    _validateAccessRestriction: validateAccessRestriction
+};
+
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    backups = require('./backups.js'),
+    BackupsError = require('./backups.js').BackupsError,
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apps'),
+    docker = require('./docker.js').connection,
+    fs = require('fs'),
+    manifestFormat = require('cloudron-manifestformat'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    shell = require('./shell.js'),
+    split = require('split'),
+    superagent = require('superagent'),
+    taskmanager = require('./taskmanager.js'),
+    util = require('util'),
+    validator = require('validator');
+
+var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
+    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+            callback();
+        });
+    };
+}
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function AppsError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AppsError, Error);
+AppsError.INTERNAL_ERROR = 'Internal Error';
+AppsError.EXTERNAL_ERROR = 'External Error';
+AppsError.ALREADY_EXISTS = 'Already Exists';
+AppsError.NOT_FOUND = 'Not Found';
+AppsError.BAD_FIELD = 'Bad Field';
+AppsError.BAD_STATE = 'Bad State';
+AppsError.PORT_RESERVED = 'Port Reserved';
+AppsError.PORT_CONFLICT = 'Port Conflict';
+AppsError.BILLING_REQUIRED = 'Billing Required';
+AppsError.ACCESS_DENIED = 'Access denied';
+AppsError.USER_REQUIRED = 'User required';
+
+// Hostname validation comes from RFC 1123 (section 2.1)
+// Domain name validation comes from RFC 2181 (Name syntax)
+// https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
+// We are validating the validity of the location-fqdn as host name
+function validateHostname(location, fqdn) {
+    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION ];
+
+    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new Error(location + ' is reserved');
+
+    if (location === '') return null; // bare location
+
+    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new Error('Hostname length cannot be greater than 63');
+    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new Error('Hostname can only contain alphanumerics and hyphen');
+    if (location[0] === '-' || location[location.length-1] === '-') return new Error('Hostname cannot start or end with hyphen');
+    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new Error('FQDN length exceeds 253 characters');
+
+    return null;
+}
+
+// validate the port bindings
+function validatePortBindings(portBindings, tcpPorts) {
+    // keep the public ports in sync with firewall rules in scripts/initializeBaseUbuntuImage.sh
+    // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
+    // for custom tcp ports
+    var RESERVED_PORTS = [
+        25, /* smtp */
+        53, /* dns */
+        80, /* http */
+        443, /* https */
+        919, /* ssh */
+        2003, /* graphite (lo) */
+        2004, /* graphite (lo) */
+        2020, /* install server */
+        config.get('port'), /* app server (lo) */
+        config.get('internalPort'), /* internal app server (lo) */
+        config.get('ldapPort'), /* ldap server (lo) */
+        config.get('oauthProxyPort'), /* oauth proxy server (lo) */
+        config.get('simpleAuthPort'), /* simple auth server (lo) */
+        3306, /* mysql (lo) */
+        8000 /* graphite (lo) */
+    ];
+
+    if (!portBindings) return null;
+
+    var env;
+    for (env in portBindings) {
+        if (!/^[a-zA-Z0-9_]+$/.test(env)) return new AppsError(AppsError.BAD_FIELD, env + ' is not valid environment variable');
+
+        if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
+        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
+
+        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, + portBindings[env]);
+    }
+
+    // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
+    // that the user wants the service disabled
+    tcpPorts = tcpPorts || { };
+    for (env in portBindings) {
+        if (!(env in tcpPorts)) return new AppsError(AppsError.BAD_FIELD, 'Invalid portBindings ' + env);
+    }
+
+    return null;
+}
+
+function validateAccessRestriction(accessRestriction) {
+    assert.strictEqual(typeof accessRestriction, 'object');
+
+    if (accessRestriction === null) return null;
+
+    if (!accessRestriction.users || !Array.isArray(accessRestriction.users)) return new Error('users array property required');
+    if (accessRestriction.users.length === 0) return new Error('users array cannot be empty');
+    if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+
+    return null;
+}
+
+function getDuplicateErrorDetails(location, portBindings, error) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(error.reason, DatabaseError.ALREADY_EXISTS);
+
+    var match = error.message.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key/);
+    if (!match) {
+        console.error('Unexpected SQL error message.', error);
+        return new AppsError(AppsError.INTERNAL_ERROR);
+    }
+
+    // check if the location conflicts
+    if (match[1] === location) return new AppsError(AppsError.ALREADY_EXISTS);
+
+    // check if any of the port bindings conflict
+    for (var env in portBindings) {
+        if (portBindings[env] === parseInt(match[1])) return new AppsError(AppsError.PORT_CONFLICT, match[1]);
+    }
+
+    return new AppsError(AppsError.ALREADY_EXISTS);
+}
+
+function getIconUrlSync(app) {
+    var iconPath = paths.APPICONS_DIR + '/' + app.id + '.png';
+    return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
+}
+
+function hasAccessTo(app, user) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof user, 'object');
+
+    if (app.accessRestriction === null) return true;
+    return app.accessRestriction.users.some(function (e) { return e === user.id; });
+}
+
+function get(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        app.iconUrl = getIconUrlSync(app);
+        app.fqdn = config.appFqdn(app.location);
+
+        callback(null, app);
+    });
+}
+
+function getBySubdomain(subdomain, callback) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getBySubdomain(subdomain, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        app.iconUrl = getIconUrlSync(app);
+        app.fqdn = config.appFqdn(app.location);
+
+        callback(null, app);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        apps.forEach(function (app) {
+            app.iconUrl = getIconUrlSync(app);
+            app.fqdn = config.appFqdn(app.location);
+        });
+
+        callback(null, apps);
+    });
+}
+
+function purchase(appStoreId, callback) {
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // Skip purchase if appStoreId is empty
+    if (appStoreId === '') return callback(null);
+
+    var url = config.apiServerOrigin() + '/api/v1/apps/' + appStoreId + '/purchase';
+
+    superagent.post(url).query({ token: config.token() }).end(function (error, res) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if (res.status === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+        if (res.status !== 201 && res.status !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
+
+        callback(null);
+    });
+}
+
+function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, icon, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert(!icon || typeof icon === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = manifestFormat.parse(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
+
+    error = checkManifestConstraints(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+
+    error = validateHostname(location, config.fqdn());
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    error = validatePortBindings(portBindings, manifest.tcpPorts);
+    if (error) return callback(error);
+
+    error = validateAccessRestriction(accessRestriction);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    // singleUser mode requires accessRestriction to contain exactly one user
+    if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
+    if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
+
+    if (icon) {
+        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+
+        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+        }
+    }
+
+    debug('Will install app with id : ' + appId);
+
+    purchase(appStoreId, function (error) {
+        if (error) return callback(error);
+
+        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, oauthProxy, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function configure(appId, location, portBindings, accessRestriction, oauthProxy, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateHostname(location, config.fqdn());
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    error = validateAccessRestriction(accessRestriction);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        error = validatePortBindings(portBindings, app.manifest.tcpPorts);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+        var values = {
+            location: location.toLowerCase(),
+            accessRestriction: accessRestriction,
+            oauthProxy: oauthProxy,
+            portBindings: portBindings,
+
+            oldConfig: {
+                location: app.location,
+                accessRestriction: app.accessRestriction,
+                portBindings: app.portBindings,
+                oauthProxy: app.oauthProxy
+            }
+        };
+
+        debug('Will configure app with id:%s values:%j', appId, values);
+
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_CONFIGURE, values, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function update(appId, force, manifest, portBindings, icon, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof force, 'boolean');
+    assert(manifest && typeof manifest === 'object');
+    assert(!portBindings || typeof portBindings === 'object');
+    assert(!icon || typeof icon === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will update app with id:%s', appId);
+
+    var error = manifestFormat.parse(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
+
+    error = checkManifestConstraints(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed:' + error.message));
+
+    error = validatePortBindings(portBindings, manifest.tcpPorts);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    if (icon) {
+        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+
+        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+        }
+    }
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        var values = {
+            manifest: manifest,
+            portBindings: portBindings,
+            oldConfig: {
+                manifest: app.manifest,
+                portBindings: app.portBindings,
+                accessRestriction: app.accessRestriction,
+                oauthProxy: app.oauthProxy
+            }
+        };
+
+        appdb.setInstallationCommand(appId, force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, portBindings, error));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function getLogStream(appId, fromLine, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof fromLine, 'number'); // behaves like tail -n
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Getting logs for %s', appId);
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
+
+        var container = docker.getContainer(app.containerId);
+        var tail = fromLine < 0 ? -fromLine : 'all';
+
+        // note: cannot access docker file directly because it needs root access
+        container.logs({ stdout: true, stderr: true, follow: true, timestamps: true, tail: tail }, function (error, logStream) {
+            if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            var lineCount = 0;
+            var skipLinesStream = split(function mapper(line) {
+                if (++lineCount < fromLine) return undefined;
+                var timestamp = line.substr(0, line.indexOf(' ')); // sometimes this has square brackets around it
+                return JSON.stringify({ lineNumber: lineCount, timestamp: timestamp.replace(/[[\]]/g,''), log: line.substr(timestamp.length + 1) });
+            });
+            skipLinesStream.close = logStream.req.abort;
+            logStream.pipe(skipLinesStream);
+            return callback(null, skipLinesStream);
+        });
+    });
+}
+
+function getLogs(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Getting logs for %s', appId);
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
+
+        var container = docker.getContainer(app.containerId);
+        // note: cannot access docker file directly because it needs root access
+        container.logs({ stdout: true, stderr: true, follow: false, timestamps: true, tail: 'all' }, function (error, logStream) {
+            if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            return callback(null, logStream);
+        });
+    });
+}
+
+function restore(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will restore app with id:%s', appId);
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        // restore without a backup is the same as re-install
+        var restoreConfig = app.lastBackupConfig, values = { };
+        if (restoreConfig) {
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+
+            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
+            if (error) return callback(error);
+
+            // ## should probably query new location, access restriction from user
+            values = {
+                manifest: restoreConfig.manifest,
+                portBindings: restoreConfig.portBindings,
+
+                oldConfig: {
+                    location: app.location,
+                    accessRestriction: app.accessRestriction,
+                    oauthProxy: app.oauthProxy,
+                    portBindings: app.portBindings,
+                    manifest: app.manifest
+                }
+            };
+        }
+
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function uninstall(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will uninstall app with id:%s', appId);
+
+    appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UNINSTALL, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        taskmanager.restartAppTask(appId); // since uninstall is allowed from any state, kill current task
+
+        callback(null);
+    });
+}
+
+function start(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will start app with id:%s', appId);
+
+    appdb.setRunCommand(appId, appdb.RSTATE_PENDING_START, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        taskmanager.restartAppTask(appId);
+
+        callback(null);
+    });
+}
+
+function stop(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will stop app with id:%s', appId);
+
+    appdb.setRunCommand(appId, appdb.RSTATE_PENDING_STOP, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        taskmanager.restartAppTask(appId);
+
+        callback(null);
+    });
+}
+
+function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
+    if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
+        return new Error('Box version exceeds Apps maxBoxVersion');
+    }
+
+    if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, config.version())) {
+        return new Error('minBoxVersion exceeds Box version');
+    }
+
+    return null;
+}
+
+function exec(appId, options, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert(options && typeof options === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var cmd = options.cmd || [ '/bin/bash' ];
+    assert(util.isArray(cmd) && cmd.length > 0);
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        var container = docker.getContainer(app.containerId);
+
+       var execOptions = {
+            AttachStdin: true,
+            AttachStdout: true,
+            AttachStderr: true,
+            Tty: true,
+            Cmd: cmd
+        };
+
+        container.exec(execOptions, function (error, exec) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+            var startOptions = {
+                Detach: false,
+                Tty: true,
+                stdin: true // this is a dockerode option that enabled openStdin in the modem
+            };
+            exec.start(startOptions, function(error, stream) {
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                if (options.rows && options.columns) {
+                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                }
+
+                return callback(null, stream);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof lastBackupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
+    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    function canAutoupdateApp(app, newManifest) {
+        var tcpPorts = newManifest.tcpPorts || { };
+        var portBindings = app.portBindings; // this is never null
+
+        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
+        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
+        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
+
+        for (var env in tcpPorts) {
+            if (!(env in portBindings)) return new Error(env + ' is required from user');
+        }
+
+        // it's fine if one or more keys got removed
+        return null;
+    }
+
+    if (!updateInfo) return callback(null);
+
+    async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
+        get(appId, function (error, app) {
+            if (error) {
+                debug('Cannot autoupdate app %s : %s', appId, error.message);
+                return iteratorDone();
+           }
+
+            error = canAutoupdateApp(app, updateInfo[appId].manifest);
+            if (error) {
+                debug('app %s requires manual update. %s', appId, error.message);
+                return iteratorDone();
+            }
+
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+                if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);
+
+                iteratorDone(null);
+            });
+        });
+    }, callback);
+}
+
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldBackup(app, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    backups.copyLastBackup(app, function (error, newBackupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewBackup(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    backups.getBackupUrl(app, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id);
+
+        async.series([
+            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
+            addons.backupAddons.bind(null, app, addonsToBackup),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ]),
+            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
+        ], function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            callback(null, result.id);
+        });
+    });
+}
+
+function backupApp(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appConfig = null, backupFunction;
+
+    if (!canBackupApp(app)) {
+        if (!app.lastBackupId) {
+            debugApp(app, 'backupApp: cannot backup app');
+            return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy and never backed up previously'));
+        }
+
+        appConfig = app.lastBackupConfig;
+        backupFunction = reuseOldBackup.bind(null, app);
+    } else {
+        appConfig = {
+            manifest: app.manifest,
+            location: app.location,
+            portBindings: app.portBindings,
+            accessRestriction: app.accessRestriction,
+            oauthProxy: app.oauthProxy
+        };
+        backupFunction = createNewBackup.bind(null, app, addonsToBackup);
+
+        if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+            return callback(safe.error);
+        }
+    }
+
+    backupFunction(function (error, backupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'backupApp: successful id:%s', backupId);
+
+        setRestorePoint(app.id, backupId, appConfig, function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            return callback(null, backupId);
+        });
+    });
+}
+
+function backup(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(appId, function (error, app) {
+        if (error && error.reason === AppsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_BACKUP, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function restoreApp(app, addonsToRestore, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    backups.getRestoreUrl(app.lastBackupId, function (error, result) {
+        if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
+        });
+    });
+}

src/apptask.js

@@ -0,0 +1,717 @@
+#!/usr/bin/env node
+
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    startTask: startTask,
+
+    // exported for testing
+    _getFreePort: getFreePort,
+    _configureNginx: configureNginx,
+    _unconfigureNginx: unconfigureNginx,
+    _createVolume: createVolume,
+    _deleteVolume: deleteVolume,
+    _allocateOAuthProxyCredentials: allocateOAuthProxyCredentials,
+    _removeOAuthProxyCredentials: removeOAuthProxyCredentials,
+    _verifyManifest: verifyManifest,
+    _registerSubdomain: registerSubdomain,
+    _unregisterSubdomain: unregisterSubdomain,
+    _reloadNginx: reloadNginx,
+    _waitForDnsPropagation: waitForDnsPropagation
+};
+
+require('supererror')({ splatchError: true });
+
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    clientdb = require('./clientdb.js'),
+    config = require('./config.js'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apptask'),
+    docker = require('./docker.js'),
+    ejs = require('ejs'),
+    fs = require('fs'),
+    hat = require('hat'),
+    manifestFormat = require('cloudron-manifestformat'),
+    net = require('net'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    SubdomainError = require('./subdomainerror.js'),
+    subdomains = require('./subdomains.js'),
+    superagent = require('superagent'),
+    sysinfo = require('./sysinfo.js'),
+    util = require('util'),
+    uuid = require('node-uuid'),
+    _ = require('underscore');
+
+var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
+    COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
+    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
+    RELOAD_COLLECTD_CMD = path.join(__dirname, 'scripts/reloadcollectd.sh'),
+    RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh'),
+    CREATEAPPDIR_CMD = path.join(__dirname, 'scripts/createappdir.sh');
+
+function initialize(callback) {
+    database.initialize(callback);
+}
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? (app.location || '(bare)') : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+// We expect conflicts to not happen despite closing the port (parallel app installs, app update does not reconfigure nginx etc)
+// https://tools.ietf.org/html/rfc6056#section-3.5 says linux uses random ephemeral port allocation
+function getFreePort(callback) {
+    var server = net.createServer();
+    server.listen(0, function () {
+        var port = server.address().port;
+        server.close(function () {
+            return callback(null, port);
+        });
+    });
+}
+
+function reloadNginx(callback) {
+    shell.sudo('reloadNginx', [ RELOAD_NGINX_CMD ], callback);
+}
+
+function configureNginx(app, callback) {
+    getFreePort(function (error, freePort) {
+        if (error) return callback(error);
+
+        var sourceDir = path.resolve(__dirname, '..');
+        var endpoint = app.oauthProxy ? 'oauthproxy' : 'app';
+        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, { sourceDir: sourceDir, adminOrigin: config.adminOrigin(), vhost: config.appFqdn(app.location), port: freePort, endpoint: endpoint });
+
+        var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
+        debugApp(app, 'writing config to %s', nginxConfigFilename);
+
+        if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
+            debugApp(app, 'Error creating nginx config : %s', safe.error.message);
+            return callback(safe.error);
+        }
+
+        async.series([
+            exports._reloadNginx,
+            updateApp.bind(null, app, { httpPort: freePort })
+        ], callback);
+    });
+}
+
+function unconfigureNginx(app, callback) {
+    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
+    if (!safe.fs.unlinkSync(nginxConfigFilename)) {
+        debugApp(app, 'Error removing nginx configuration : %s', safe.error.message);
+        return callback(null);
+    }
+
+    exports._reloadNginx(callback);
+}
+
+function createContainer(app, callback) {
+    assert(!app.containerId); // otherwise, it will trigger volumeFrom
+
+    debugApp(app, 'creating container');
+
+    docker.createContainer(app, function (error, container) {
+        if (error) return callback(new Error('Error creating container: ' + error));
+
+        updateApp(app, { containerId: container.id }, callback);
+    });
+}
+
+function deleteContainers(app, callback) {
+    debugApp(app, 'deleting containers');
+
+    docker.deleteContainers(app.id, function (error) {
+        if (error) return callback(new Error('Error deleting container: ' + error));
+
+        updateApp(app, { containerId: null }, callback);
+    });
+}
+
+function createVolume(app, callback) {
+    shell.sudo('createVolume', [ CREATEAPPDIR_CMD, app.id ], callback);
+}
+
+function deleteVolume(app, callback) {
+    shell.sudo('deleteVolume', [ RMAPPDIR_CMD, app.id ], callback);
+}
+
+function allocateOAuthProxyCredentials(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!app.oauthProxy) return callback(null);
+
+    var id = 'cid-' + uuid.v4();
+    var clientSecret = hat(256);
+    var redirectURI = 'https://' + config.appFqdn(app.location);
+    var scope = 'profile';
+
+    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+}
+
+function removeOAuthProxyCredentials(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) {
+            debugApp(app, 'Error removing OAuth client id', error);
+            return callback(error);
+        }
+
+        callback(null);
+    });
+}
+
+function addCollectdProfile(app, callback) {
+    var collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId });
+    fs.writeFile(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), collectdConf, function (error) {
+        if (error) return callback(error);
+        shell.sudo('addCollectdProfile', [ RELOAD_COLLECTD_CMD ], callback);
+    });
+}
+
+function removeCollectdProfile(app, callback) {
+    fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), function (error) {
+        if (error && error.code !== 'ENOENT') debugApp(app, 'Error removing collectd profile', error);
+        shell.sudo('removeCollectdProfile', [ RELOAD_COLLECTD_CMD ], callback);
+    });
+}
+
+function verifyManifest(app, callback) {
+    debugApp(app, 'Verifying manifest');
+
+    var manifest = app.manifest;
+    var error = manifestFormat.parse(manifest);
+    if (error) return callback(new Error(util.format('Manifest error: %s', error.message)));
+
+    error = apps.checkManifestConstraints(manifest);
+    if (error) return callback(error);
+
+    return callback(null);
+}
+
+function downloadIcon(app, callback) {
+    debugApp(app, 'Downloading icon of %s@%s', app.appStoreId, app.manifest.version);
+
+    var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
+
+    superagent
+        .get(iconUrl)
+        .buffer(true)
+        .end(function (error, res) {
+            if (error) return callback(new Error('Error downloading icon:' + error.message));
+            if (res.status !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+
+            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
+
+            callback(null);
+    });
+}
+
+function registerSubdomain(app, callback) {
+    // even though the bare domain is already registered in the appstore, we still
+    // need to register it so that we have a dnsRecordId to wait for it to complete
+    var record = { subdomain: app.location, type: 'A', value: sysinfo.getIp() };
+
+    async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Registering subdomain location [%s]', app.location);
+
+        subdomains.add(record, function (error, changeId) {
+            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+            retryCallback(null, error || changeId);
+        });
+    }, function (error, result) {
+        if (error || result instanceof Error) return callback(error || result);
+
+        updateApp(app, { dnsRecordId: result }, callback);
+    });
+}
+
+function unregisterSubdomain(app, location, callback) {
+    // do not unregister bare domain because we show a error/cloudron info page there
+    if (location === '') {
+        debugApp(app, 'Skip unregister of empty subdomain');
+        return callback(null);
+    }
+
+    var record = { subdomain: location, type: 'A', value: sysinfo.getIp() };
+
+    async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+        debugApp(app, 'Unregistering subdomain: %s', location);
+
+        subdomains.remove(record, function (error) {
+            if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR))return retryCallback(error); // try again
+
+            retryCallback(error);
+        });
+    }, function (error) {
+        if (error) debugApp(app, 'Error unregistering subdomain: %s', error);
+
+        updateApp(app, { dnsRecordId: null }, callback);
+    });
+}
+
+function removeIcon(app, callback) {
+    fs.unlink(path.join(paths.APPICONS_DIR, app.id + '.png'), function (error) {
+        if (error && error.code !== 'ENOENT') debugApp(app, 'cannot remove icon : %s', error);
+        callback(null);
+    });
+}
+
+function waitForDnsPropagation(app, callback) {
+    if (!config.CLOUDRON) {
+        debugApp(app, 'Skipping dns propagation check for development');
+        return callback(null);
+    }
+
+    function retry(error) {
+        debugApp(app, 'waitForDnsPropagation: ', error);
+        setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
+    }
+
+    subdomains.status(app.dnsRecordId, function (error, result) {
+        if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
+
+        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
+
+        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
+
+        callback(null);
+    });
+}
+
+// updates the app object and the database
+function updateApp(app, values, callback) {
+    debugApp(app, 'updating app with values: %j', values);
+
+    appdb.update(app.id, values, function (error) {
+        if (error) return callback(error);
+
+        for (var value in values) {
+            app[value] = values[value];
+        }
+
+        return callback(null);
+    });
+}
+
+// Ordering is based on the following rationale:
+//   - configure nginx, icon, oauth
+//   - register subdomain.
+//          at this point, the user can visit the site and the above nginx config can show some install screen.
+//          the icon can be displayed in this nginx page and oauth proxy means the page can be protected
+//   - download image
+//   - setup volumes
+//   - setup addons (requires the above volume)
+//   - setup the container (requires image, volumes, addons)
+//   - setup collectd (requires container id)
+function install(app, callback) {
+    async.series([
+        verifyManifest.bind(null, app),
+
+        // teardown for re-installs
+        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        removeCollectdProfile.bind(null, app),
+        stopApp.bind(null, app),
+        deleteContainers.bind(null, app),
+        addons.teardownAddons.bind(null, app, app.manifest.addons),
+        deleteVolume.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
+        removeOAuthProxyCredentials.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
+        unconfigureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
+        configureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '20, Downloading icon' }),
+        downloadIcon.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '25, Creating OAuth proxy credentials' }),
+        allocateOAuthProxyCredentials.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '30, Registering subdomain' }),
+        registerSubdomain.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '40, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
+        updateApp.bind(null, app, { installationProgress: '50, Creating volume' }),
+        createVolume.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '60, Setting up addons' }),
+        addons.setupAddons.bind(null, app, app.manifest.addons),
+
+        updateApp.bind(null, app, { installationProgress: '70, Creating container' }),
+        createContainer.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '80, Setting up collectd profile' }),
+        addCollectdProfile.bind(null, app),
+
+        runApp.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        exports._waitForDnsPropagation.bind(null, app),
+
+        // done!
+        function (callback) {
+            debugApp(app, 'installed');
+            updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '', health: null }, callback);
+        }
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error installing app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
+function backup(app, callback) {
+    async.series([
+        updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
+        apps.backupApp.bind(null, app, app.manifest.addons),
+
+        // done!
+        function (callback) {
+            debugApp(app, 'installed');
+            updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '' }, callback);
+        }
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error backing up app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: error.message }, callback.bind(null, error)); // return to installed state intentionally
+        }
+        callback(null);
+    });
+}
+
+// restore is also called for upgrades and infra updates. note that in those cases it is possible there is no backup
+function restore(app, callback) {
+    // we don't have a backup, same as re-install. this allows us to install from install failures (update failures always
+    // have a backupId)
+    if (!app.lastBackupId) {
+        debugApp(app, 'No lastBackupId. reinstalling');
+        return install(app, callback);
+    }
+
+    async.series([
+        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        removeCollectdProfile.bind(null, app),
+        stopApp.bind(null, app),
+        deleteContainers.bind(null, app),
+         // oldConfig can be null during upgrades
+        addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
+        deleteVolume.bind(null, app),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             docker.deleteImage(app.oldConfig.manifest, done);
+        },
+        removeOAuthProxyCredentials.bind(null, app),
+        removeIcon.bind(null, app),
+        unconfigureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '30, Configuring Nginx' }),
+        configureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '40, Downloading icon' }),
+        downloadIcon.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '50, Create OAuth proxy credentials' }),
+        allocateOAuthProxyCredentials.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '55, Registering subdomain' }), // ip might change during upgrades
+        registerSubdomain.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '60, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
+        updateApp.bind(null, app, { installationProgress: '65, Creating volume' }),
+        createVolume.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
+        apps.restoreApp.bind(null, app, app.manifest.addons),
+
+        updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
+        createContainer.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '80, Setting up collectd profile' }),
+        addCollectdProfile.bind(null, app),
+
+        runApp.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
+        exports._waitForDnsPropagation.bind(null, app),
+
+        // done!
+        function (callback) {
+            debugApp(app, 'restored');
+            updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '', health: null }, callback);
+        }
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'Error installing app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+
+        callback(null);
+    });
+}
+
+// note that configure is called after an infra update as well
+function configure(app, callback) {
+    async.series([
+        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        removeCollectdProfile.bind(null, app),
+        stopApp.bind(null, app),
+        deleteContainers.bind(null, app),
+        function (next) {
+            // oldConfig can be null during an infra update
+            if (!app.oldConfig || app.oldConfig.location === app.location) return next();
+            unregisterSubdomain(app, app.oldConfig.location, next);
+        },
+        removeOAuthProxyCredentials.bind(null, app),
+        unconfigureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '25, Configuring Nginx' }),
+        configureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
+        allocateOAuthProxyCredentials.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
+        registerSubdomain.bind(null, app),
+
+        // re-setup addons since they rely on the app's fqdn (e.g oauth)
+        updateApp.bind(null, app, { installationProgress: '50, Setting up addons' }),
+        addons.setupAddons.bind(null, app, app.manifest.addons),
+
+        updateApp.bind(null, app, { installationProgress: '60, Creating container' }),
+        createContainer.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '70, Add collectd profile' }),
+        addCollectdProfile.bind(null, app),
+
+        runApp.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
+        exports._waitForDnsPropagation.bind(null, app),
+
+        // done!
+        function (callback) {
+            debugApp(app, 'configured');
+            updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '', health: null }, callback);
+        }
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error reconfiguring : %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
+// nginx configuration is skipped because app.httpPort is expected to be available
+function update(app, callback) {
+    debugApp(app, 'Updating to %s', safe.query(app, 'manifest.version'));
+
+    // app does not want these addons anymore
+    var unusedAddons = _.omit(app.oldConfig.manifest.addons, Object.keys(app.manifest.addons));
+
+    async.series([
+        updateApp.bind(null, app, { installationProgress: '0, Verify manifest' }),
+        verifyManifest.bind(null, app),
+
+        // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
+        // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
+        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        removeCollectdProfile.bind(null, app),
+        stopApp.bind(null, app),
+        deleteContainers.bind(null, app),
+        addons.teardownAddons.bind(null, app, unusedAddons),
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             docker.deleteImage(app.oldConfig.manifest, done);
+        },
+        // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
+
+        function (next) {
+            if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);
+
+            async.series([
+                updateApp.bind(null, app, { installationProgress: '20, Backup app' }),
+                apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+            ], next);
+        },
+
+        updateApp.bind(null, app, { installationProgress: '35, Downloading icon' }),
+        downloadIcon.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '45, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
+        updateApp.bind(null, app, { installationProgress: '70, Updating addons' }),
+        addons.setupAddons.bind(null, app, app.manifest.addons),
+
+        updateApp.bind(null, app, { installationProgress: '80, Creating container' }),
+        createContainer.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '90, Add collectd profile' }),
+        addCollectdProfile.bind(null, app),
+
+        runApp.bind(null, app),
+
+        // done!
+        function (callback) {
+            debugApp(app, 'updated');
+            updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '', health: null }, callback);
+        }
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'Error updating app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
+function uninstall(app, callback) {
+    debugApp(app, 'uninstalling');
+
+    async.series([
+        updateApp.bind(null, app, { installationProgress: '0, Remove collectd profile' }),
+        removeCollectdProfile.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '10, Stopping app' }),
+        stopApp.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '20, Deleting container' }),
+        deleteContainers.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '30, Teardown addons' }),
+        addons.teardownAddons.bind(null, app, app.manifest.addons),
+
+        updateApp.bind(null, app, { installationProgress: '40, Deleting volume' }),
+        deleteVolume.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '50, Deleting image' }),
+        docker.deleteImage.bind(null, app.manifest),
+
+        updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
+        unregisterSubdomain.bind(null, app, app.location),
+
+        updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
+        removeOAuthProxyCredentials.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '80, Cleanup icon' }),
+        removeIcon.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '90, Unconfiguring Nginx' }),
+        unconfigureNginx.bind(null, app),
+
+        updateApp.bind(null, app, { installationProgress: '95, Remove app from database' }),
+        appdb.del.bind(null, app.id)
+    ], callback);
+}
+
+function runApp(app, callback) {
+    docker.startContainer(app.containerId, function (error) {
+        if (error) return callback(error);
+
+        updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
+    });
+}
+
+function stopApp(app, callback) {
+    docker.stopContainers(app.id, function (error) {
+        if (error) return callback(error);
+
+        updateApp(app, { runState: appdb.RSTATE_STOPPED }, callback);
+    });
+}
+
+function handleRunCommand(app, callback) {
+    if (app.runState === appdb.RSTATE_PENDING_STOP) {
+        return stopApp(app, callback);
+    }
+
+    if (app.runState === appdb.RSTATE_PENDING_START || app.runState === appdb.RSTATE_RUNNING) {
+        debugApp(app, 'Resuming app with state : %s', app.runState);
+        return runApp(app, callback);
+    }
+
+    debugApp(app, 'handleRunCommand - doing nothing: %s', app.runState);
+
+    return callback(null);
+}
+
+function startTask(appId, callback) {
+    // determine what to do
+    appdb.get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        debugApp(app, 'startTask installationState: %s runState: %s', app.installationState, app.runState);
+
+        switch (app.installationState) {
+        case appdb.ISTATE_PENDING_UNINSTALL: return uninstall(app, callback);
+        case appdb.ISTATE_PENDING_CONFIGURE: return configure(app, callback);
+        case appdb.ISTATE_PENDING_UPDATE: return update(app, callback);
+        case appdb.ISTATE_PENDING_RESTORE: return restore(app, callback);
+        case appdb.ISTATE_PENDING_BACKUP: return backup(app, callback);
+        case appdb.ISTATE_INSTALLED: return handleRunCommand(app, callback);
+        case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
+        case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
+        case appdb.ISTATE_ERROR:
+            debugApp(app, 'Apptask launched with error states.');
+            return callback(null);
+        default:
+            debugApp(app, 'apptask launched with invalid command');
+            return callback(new Error('Unknown command in apptask:' + app.installationState));
+        }
+    });
+}
+
+if (require.main === module) {
+    assert.strictEqual(process.argv.length, 3, 'Pass the appid as argument');
+
+    debug('Apptask for %s', process.argv[2]);
+
+    initialize(function (error) {
+        if (error) throw error;
+
+        startTask(process.argv[2], function (error) {
+            if (error) console.error(error);
+
+            debug('Apptask completed for %s', process.argv[2]);
+            // https://nodejs.org/api/process.html are exit codes used by node. apps.js uses the value below
+            // to check apptask crashes
+            process.exit(error ? 50 : 0);
+        });
+    });
+}
+

src/auth.js

@@ -0,0 +1,139 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize
+};
+
+var assert = require('assert'),
+    BasicStrategy = require('passport-http').BasicStrategy,
+    BearerStrategy = require('passport-http-bearer').Strategy,
+    clientdb = require('./clientdb'),
+    ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    DatabaseError = require('./databaseerror'),
+    debug = require('debug')('box:auth'),
+    LocalStrategy = require('passport-local').Strategy,
+    crypto = require('crypto'),
+    passport = require('passport'),
+    tokendb = require('./tokendb'),
+    user = require('./user'),
+    userdb = require('./userdb'),
+    UserError = user.UserError,
+    _ = require('underscore');
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    passport.serializeUser(function (user, callback) {
+        callback(null, user.username);
+    });
+
+    passport.deserializeUser(function(username, callback) {
+        userdb.get(username, function (error, result) {
+            if (error) return callback(error);
+
+            var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
+            result.gravatar = 'https://www.gravatar.com/avatar/' + md5 + '.jpg?s=24&d=mm';
+
+            callback(null, result);
+        });
+    });
+
+    passport.use(new LocalStrategy(function (username, password, callback) {
+        if (username.indexOf('@') === -1) {
+            user.verify(username, password, function (error, result) {
+                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, _.pick(result, 'id', 'username', 'email', 'admin'));
+            });
+        } else {
+            user.verifyWithEmail(username, password, function (error, result) {
+                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, _.pick(result, 'id', 'username', 'email', 'admin'));
+            });
+        }
+    }));
+
+    passport.use(new BasicStrategy(function (username, password, callback) {
+        if (username.indexOf('cid-') === 0) {
+            debug('BasicStrategy: detected client id %s instead of username:password', username);
+            // username is actually client id here
+            // password is client secret
+            clientdb.get(username, function (error, client) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+                if (error) return callback(error);
+                if (client.clientSecret != password) return callback(null, false);
+                return callback(null, client);
+            });
+        } else {
+            user.verify(username, password, function (error, result) {
+                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, result);
+            });
+        }
+    }));
+
+    passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
+        clientdb.get(clientId, function(error, client) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            if (error) { return callback(error); }
+            if (client.clientSecret != clientSecret) { return callback(null, false); }
+            return callback(null, client);
+        });
+    }));
+
+    passport.use(new BearerStrategy(function (accessToken, callback) {
+        tokendb.get(accessToken, function (error, token) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            if (error) return callback(error);
+
+            // scopes here can define what capabilities that token carries
+            // passport put the 'info' object into req.authInfo, where we can further validate the scopes
+            var info = { scope: token.scope };
+            var tokenType;
+
+            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
+                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
+                tokenType = tokendb.TYPE_DEV;
+            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
+                tokenType = tokendb.TYPE_APP;
+                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
+            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
+                tokenType = tokendb.TYPE_USER;
+                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
+            } else {
+                // legacy tokens assuming a user access token
+                tokenType = tokendb.TYPE_USER;
+            }
+
+            userdb.get(token.identifier, function (error, user) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+                if (error) return callback(error);
+
+                // amend the tokenType of the token owner
+                user.tokenType = tokenType;
+
+                callback(null, user, info);
+            });
+        });
+    }));
+
+    callback(null);
+}
+
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    callback(null);
+}
+

src/authcodedb.js

@@ -0,0 +1,78 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    add: add,
+    del: del,
+    delExpired: delExpired,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror');
+
+var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId', 'expiresAt' ].join(',');
+
+function get(authCode, callback) {
+    assert.strictEqual(typeof authCode, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ? AND expiresAt > ?', [ authCode, Date.now() ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result[0]);
+    });
+}
+
+function add(authCode, clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof authCode, 'string');
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO authcodes (authCode, clientId, userId, expiresAt) VALUES (?, ?, ?, ?)',
+            [ authCode, clientId, userId, expiresAt ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(authCode, callback) {
+    assert.strictEqual(typeof authCode, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function delExpired(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM authcodes WHERE expiresAt <= ?', [ Date.now() ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        return callback(null, result.affectedRows);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM authcodes', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+

src/aws.js

@@ -0,0 +1,282 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    getSignedUploadUrl: getSignedUploadUrl,
+    getSignedDownloadUrl: getSignedDownloadUrl,
+
+    addSubdomain: addSubdomain,
+    delSubdomain: delSubdomain,
+    getChangeStatus: getChangeStatus,
+
+    copyObject: copyObject
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('./config.js'),
+    debug = require('debug')('box:aws'),
+    SubdomainError = require('./subdomainerror.js'),
+    superagent = require('superagent');
+
+function getAWSCredentials(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // CaaS
+    if (config.token()) {
+        var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/awscredentials';
+        superagent.post(url).query({ token: config.token() }).end(function (error, result) {
+            if (error) return callback(error);
+            if (result.statusCode !== 201) return callback(new Error(result.text));
+            if (!result.body || !result.body.credentials) return callback(new Error('Unexpected response'));
+
+            var credentials = {
+                accessKeyId: result.body.credentials.AccessKeyId,
+                secretAccessKey: result.body.credentials.SecretAccessKey,
+                sessionToken: result.body.credentials.SessionToken,
+                region: 'us-east-1'
+            };
+
+            if (config.aws().endpoint) credentials.endpoint = new AWS.Endpoint(config.aws().endpoint);
+
+            callback(null, credentials);
+        });
+    } else {
+        if (!config.aws().accessKeyId || !config.aws().secretAccessKey) return callback(new SubdomainError(SubdomainError.MISSING_CREDENTIALS));
+
+        var credentials = {
+            accessKeyId: config.aws().accessKeyId,
+            secretAccessKey: config.aws().secretAccessKey,
+            region: 'us-east-1'
+        };
+
+        if (config.aws().endpoint) credentials.endpoint = new AWS.Endpoint(config.aws().endpoint);
+
+        callback(null, credentials);
+    }
+}
+
+function getSignedUploadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedUploadUrl: %s', filename);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('putObject', params);
+
+        callback(null, { url : url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getSignedDownloadUrl(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getSignedDownloadUrl: %s', filename);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var s3 = new AWS.S3(credentials);
+
+        var params = {
+            Bucket: config.aws().backupBucket,
+            Key: config.aws().backupPrefix + '/' + filename,
+            Expires: 60 * 30 /* 30 minutes */
+        };
+
+        var url = s3.getSignedUrl('getObject', params);
+
+        callback(null, { url: url, sessionToken: credentials.sessionToken });
+    });
+}
+
+function getZoneByName(zoneName, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getZoneByName: %s', zoneName);
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.listHostedZones({}, function (error, result) {
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+
+            var zone = result.HostedZones.filter(function (zone) {
+                return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+            })[0];
+
+            if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
+
+            debug('getZoneByName: found zone', zone);
+
+            callback(null, zone);
+        });
+    });
+}
+
+function addSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('addSubdomain: ' + subdomain + ' for domain ' + zoneName + ' with value ' + value);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'UPSERT',
+                    ResourceRecordSet: {
+                        Type: type,
+                        Name: fqdn,
+                        ResourceRecords: [{
+                            Value: value
+                        }],
+                        Weight: 0,
+                        SetIdentifier: fqdn,
+                        TTL: 1
+                    }
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.code === 'PriorRequestNotComplete') {
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+                } else if (error) {
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+                }
+
+                debug('addSubdomain: success. changeInfoId:%j', result);
+
+                callback(null, result.ChangeInfo.Id);
+            });
+        });
+    });
+}
+
+function delSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('delSubdomain: %s for domain %s.', subdomain, zoneName);
+
+    getZoneByName(zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var resourceRecordSet = {
+            Name: fqdn,
+            Type: type,
+            ResourceRecords: [{
+                Value: value
+            }],
+            Weight: 0,
+            SetIdentifier: fqdn,
+            TTL: 1
+        };
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'DELETE',
+                    ResourceRecordSet: resourceRecordSet
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        getAWSCredentials(function (error, credentials) {
+            if (error) return callback(error);
+
+            var route53 = new AWS.Route53(credentials);
+            route53.changeResourceRecordSets(params, function(error, result) {
+                if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                    debug('delSubdomain: resource record set not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'NoSuchHostedZone') {
+                    debug('delSubdomain: hosted zone not found.', error);
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error && error.code === 'PriorRequestNotComplete') {
+                    debug('delSubdomain: resource is still busy', error);
+                    return callback(new SubdomainError(SubdomainError.STILL_BUSY, new Error(error)));
+                } else if (error && error.code === 'InvalidChangeBatch') {
+                    debug('delSubdomain: invalid change batch. No such record to be deleted.');
+                    return callback(new SubdomainError(SubdomainError.NOT_FOUND, new Error(error)));
+                } else if (error) {
+                    debug('delSubdomain: error', error);
+                    return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, new Error(error)));
+                }
+
+                debug('delSubdomain: success');
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function getChangeStatus(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var route53 = new AWS.Route53(credentials);
+        route53.getChange({ Id: changeId }, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, result.ChangeInfo.Status);
+        });
+    });
+}
+
+function copyObject(from, to, callback) {
+    assert.strictEqual(typeof from, 'string');
+    assert.strictEqual(typeof to, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getAWSCredentials(function (error, credentials) {
+        if (error) return callback(error);
+
+        var params = {
+            Bucket: config.aws().backupBucket, // target bucket
+            Key: config.aws().backupPrefix + '/' + to, // target file
+            CopySource: config.aws().backupBucket + '/' + config.aws().backupPrefix + '/' + from, // source
+        };
+
+        var s3 = new AWS.S3(credentials);
+        s3.copyObject(params, callback);
+    });
+}

src/backups.js

@@ -0,0 +1,119 @@
+'use strict';
+
+exports = module.exports = {
+    BackupsError: BackupsError,
+
+    getAllPaged: getAllPaged,
+
+    getBackupUrl: getBackupUrl,
+    getRestoreUrl: getRestoreUrl,
+
+    copyLastBackup: copyLastBackup
+};
+
+var assert = require('assert'),
+    aws = require('./aws.js'),
+    config = require('./config.js'),
+    debug = require('debug')('box:backups'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function BackupsError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(BackupsError, Error);
+BackupsError.EXTERNAL_ERROR = 'external error';
+BackupsError.INTERNAL_ERROR = 'internal error';
+
+function getAllPaged(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+
+    superagent.get(url).query({ token: config.token() }).end(function (error, result) {
+        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+        if (!result.body || !util.isArray(result.body.backups)) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+
+        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
+        return callback(null, result.body.backups);
+    });
+}
+
+function getBackupUrl(app, callback) {
+    assert(!app || typeof app === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var filename = '';
+    if (app) {
+        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    } else {
+        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
+    }
+
+    aws.getSignedUploadUrl(filename, function (error, result) {
+        if (error) return callback(error);
+
+        var obj = {
+            id: filename,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
+
+        debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+        callback(null, obj);
+    });
+}
+
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+function getRestoreUrl(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    aws.getSignedDownloadUrl(backupId, function (error, result) {
+        if (error) return callback(error);
+
+        var obj = {
+            id: backupId,
+            url: result.url,
+            sessionToken: result.sessionToken,
+            backupKey: config.backupKey()
+        };
+
+        debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+        callback(null, obj);
+    });
+}
+
+function copyLastBackup(app, callback) {
+    assert(app && typeof app === 'object');
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    aws.copyObject(app.lastBackupId, toFilename, function (error) {
+        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+        return callback(null, toFilename);
+    });
+}

src/caas.js

@@ -0,0 +1,91 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    addSubdomain: addSubdomain,
+    delSubdomain: delSubdomain,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:caas'),
+    SubdomainError = require('./subdomainerror.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function addSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var fqdn = subdomain !== '' && type === 'TXT' ? subdomain + '.' + config.fqdn() : config.appFqdn(subdomain);
+
+    debug('addSubdomain: zoneName: %s subdomain: %s type: %s value: %s fqdn: %s', zoneName, subdomain, type, value, fqdn);
+
+    var data = {
+        type: type,
+        value: value
+    };
+
+    superagent
+        .post(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
+        .query({ token: config.token() })
+        .send(data)
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.status !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.changeId);
+        });
+}
+
+function delSubdomain(zoneName, subdomain, type, value, callback) {
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('delSubdomain: %s for domain %s.', subdomain, zoneName);
+
+    var data = {
+        type: type,
+        value: value
+    };
+
+    superagent
+        .del(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
+        .query({ token: config.token() })
+        .send(data)
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.status === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND));
+            if (result.status !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null);
+        });
+}
+
+function getChangeStatus(changeId, callback) {
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
+        .query({ token: config.token() })
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return callback(null, result.body.status);
+        });
+
+}

src/clientdb.js

@@ -0,0 +1,156 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getAll: getAll,
+    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
+    add: add,
+    del: del,
+    getByAppId: getByAppId,
+    getByAppIdAndType: getByAppIdAndType,
+
+    delByAppId: delByAppId,
+    delByAppIdAndType: delByAppIdAndType,
+
+    _clear: clear,
+
+    TYPE_EXTERNAL: 'external',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy',
+    TYPE_ADMIN: 'admin'
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js');
+
+var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
+var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients ORDER BY appId', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function getAllWithTokenCountByIdentifier(identifier, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null, results);
+
+    });
+}
+
+function getByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null, result[0]);
+    });
+}
+
+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null, result[0]);
+    });
+}
+
+function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof clientSecret, 'string');
+    assert.strictEqual(typeof redirectURI, 'string');
+    assert.strictEqual(typeof scope, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
+
+    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM clients WHERE id = ?', [ id ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function delByAppId(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM clients WHERE appId=?', [ appId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+

src/clients.js

@@ -0,0 +1,186 @@
+'use strict';
+
+exports = module.exports = {
+    ClientsError: ClientsError,
+
+    add: add,
+    get: get,
+    del: del,
+    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getClientTokensByUserId: getClientTokensByUserId,
+    delClientTokensByUserId: delClientTokensByUserId
+};
+
+var assert = require('assert'),
+    util = require('util'),
+    hat = require('hat'),
+    appdb = require('./appdb.js'),
+    tokendb = require('./tokendb.js'),
+    constants = require('./constants.js'),
+    async = require('async'),
+    clientdb = require('./clientdb.js'),
+    DatabaseError = require('./databaseerror.js'),
+    uuid = require('node-uuid');
+
+function ClientsError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(ClientsError, Error);
+ClientsError.INVALID_SCOPE = 'Invalid scope';
+ClientsError.INVALID_CLIENT = 'Invalid client';
+
+function validateScope(scope) {
+    assert.strictEqual(typeof scope, 'string');
+
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
+    if (scope === '*') return null;
+
+    // TODO maybe validate all individual scopes if they exist
+
+    return null;
+}
+
+function add(appId, type, redirectURI, scope, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof redirectURI, 'string');
+    assert.strictEqual(typeof scope, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateScope(scope);
+    if (error) return callback(error);
+
+    var id = 'cid-' + uuid.v4();
+    var clientSecret = hat(256);
+
+    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
+        if (error) return callback(error);
+
+        var client = {
+            id: id,
+            appId: appId,
+            type: type,
+            clientSecret: clientSecret,
+            redirectURI: redirectURI,
+            scope: scope
+        };
+
+        callback(null, client);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.get(id, function (error, result) {
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.del(id, function (error, result) {
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
+function getAllWithDetailsByUserId(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
+        if (error) return callback(error);
+
+        var tmp = [];
+        async.each(results, function (record, callback) {
+            if (record.type === clientdb.TYPE_ADMIN) {
+                record.name = constants.ADMIN_NAME;
+                record.location = constants.ADMIN_LOCATION;
+
+                tmp.push(record);
+
+                return callback(null);
+            }
+
+            appdb.get(record.appId, function (error, result) {
+                if (error) {
+                    console.error('Failed to get app details for oauth client', result, error);
+                    return callback(null);  // ignore error so we continue listing clients
+                }
+
+                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
+                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+
+                record.location = result.location;
+
+                tmp.push(record);
+
+                callback(null);
+            });
+        }, function (error) {
+            if (error) return callback(error);
+            callback(null, tmp);
+        });
+    });
+}
+
+function getClientTokensByUserId(clientId, userId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) {
+            // this can mean either that there are no tokens or the clientId is actually unknown
+            clientdb.get(clientId, function (error/*, result*/) {
+                if (error) return callback(error);
+                callback(null, []);
+            });
+            return;
+        }
+        if (error) return callback(error);
+        callback(null, result || []);
+    });
+}
+
+function delClientTokensByUserId(clientId, userId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) {
+            // this can mean either that there are no tokens or the clientId is actually unknown
+            clientdb.get(clientId, function (error/*, result*/) {
+                if (error) return callback(error);
+                callback(null);
+            });
+            return;
+        }
+        if (error) return callback(error);
+        callback(null);
+    });
+}

src/cloudron.js

@@ -0,0 +1,649 @@
+/* jslint node: true */
+
+'use strict';
+
+exports = module.exports = {
+    CloudronError: CloudronError,
+
+    initialize: initialize,
+    uninitialize: uninitialize,
+    activate: activate,
+    getConfig: getConfig,
+    getStatus: getStatus,
+
+    setCertificate: setCertificate,
+
+    sendHeartbeat: sendHeartbeat,
+
+    update: update,
+    reboot: reboot,
+    migrate: migrate,
+    backup: backup,
+    ensureBackup: ensureBackup
+};
+
+var apps = require('./apps.js'),
+    AppsError = require('./apps.js').AppsError,
+    assert = require('assert'),
+    async = require('async'),
+    backups = require('./backups.js'),
+    BackupsError = require('./backups.js').BackupsError,
+    bytes = require('bytes'),
+    clientdb = require('./clientdb.js'),
+    config = require('./config.js'),
+    debug = require('debug')('box:cloudron'),
+    fs = require('fs'),
+    locker = require('./locker.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    progress = require('./progress.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
+    superagent = require('superagent'),
+    sysinfo = require('./sysinfo.js'),
+    tokendb = require('./tokendb.js'),
+    updateChecker = require('./updatechecker.js'),
+    user = require('./user.js'),
+    UserError = user.UserError,
+    userdb = require('./userdb.js'),
+    util = require('util'),
+    webhooks = require('./webhooks.js');
+
+var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
+    REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
+    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
+    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
+
+var gAddDnsRecordsTimerId = null,
+    gCloudronDetails = null;            // cached cloudron details like region,size...
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+            callback();
+        });
+    };
+}
+
+
+function CloudronError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(CloudronError, Error);
+CloudronError.BAD_FIELD = 'Field error';
+CloudronError.INTERNAL_ERROR = 'Internal Error';
+CloudronError.EXTERNAL_ERROR = 'External Error';
+CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
+CloudronError.BAD_USERNAME = 'Bad username';
+CloudronError.BAD_EMAIL = 'Bad email';
+CloudronError.BAD_PASSWORD = 'Bad password';
+CloudronError.BAD_NAME = 'Bad name';
+CloudronError.BAD_STATE = 'Bad state';
+CloudronError.NOT_FOUND = 'Not found';
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (process.env.BOX_ENV !== 'test') {
+        addDnsRecords();
+    }
+
+    callback(null);
+}
+
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    clearTimeout(gAddDnsRecordsTimerId);
+    gAddDnsRecordsTimerId = null;
+
+    callback(null);
+}
+
+function setTimeZone(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('setTimeZone ip:%s', ip);
+
+    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
+        if (error || result.statusCode !== 200) {
+            debug('Failed to get geo location', error);
+            return callback(null);
+        }
+
+        if (!result.body.timezone) {
+            debug('No timezone in geoip response : %j', result.body);
+            return callback(null);
+        }
+
+        debug('Setting timezone to ', result.body.timezone);
+
+        settings.setTimeZone(result.body.timezone, callback);
+    });
+}
+
+function activate(username, password, email, ip, callback) {
+    assert.strictEqual(typeof username, 'string');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('activating user:%s email:%s', username, email);
+
+    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
+
+    user.createOwner(username, password, email, function (error, userObject) {
+        if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
+        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
+        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
+        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            // Also generate a token so the admin creation can also act as a login
+            var token = tokendb.generateToken();
+            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+
+            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+                callback(null, { token: token, expires: expires });
+            });
+        });
+    });
+}
+
+function getStatus(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    userdb.count(function (error, count) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        settings.getCloudronName(function (error, cloudronName) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                activated: count !== 0,
+                version: config.version(),
+                cloudronName: cloudronName
+            });
+        });
+    });
+}
+
+function getCloudronDetails(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gCloudronDetails) return callback(null, gCloudronDetails);
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
+        .query({ token: config.token() })
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            gCloudronDetails = result.body.box;
+
+            return callback(null, gCloudronDetails);
+        });
+}
+
+function getConfig(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getCloudronDetails(function (error, result) {
+        if (error) {
+            console.error('Failed to fetch cloudron details.', error);
+
+            // set fallback values to avoid dependency on appstore
+            result = {
+                region: result ? result.region : null,
+                size: result ? result.size : null
+            };
+        }
+
+        // We rely at the moment on the size being specified in 512mb,1gb,...
+        // TODO provide that number from the appstore
+        var memory = bytes(result.size) || 0;
+
+        settings.getCloudronName(function (error, cloudronName) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            settings.getDeveloperMode(function (error, developerMode) {
+                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+                callback(null, {
+                    apiServerOrigin: config.apiServerOrigin(),
+                    webServerOrigin: config.webServerOrigin(),
+                    isDev: config.isDev(),
+                    fqdn: config.fqdn(),
+                    ip: sysinfo.getIp(),
+                    version: config.version(),
+                    update: updateChecker.getUpdateInfo(),
+                    progress: progress.get(),
+                    isCustomDomain: config.isCustomDomain(),
+                    developerMode: developerMode,
+                    region: result.region,
+                    size: result.size,
+                    memory: memory,
+                    cloudronName: cloudronName
+                });
+            });
+        });
+    });
+}
+
+function sendHeartbeat() {
+    // Only send heartbeats after the admin dns record is synced to give appstore a chance to know that fact
+    if (!config.dnsInSync()) return;
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
+
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
+        if (error) debug('Error sending heartbeat.', error);
+        else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
+        else debug('Heartbeat sent to %s', url);
+    });
+}
+
+function addDnsRecords() {
+    if (config.dnsInSync()) return sendHeartbeat(); // already registered send heartbeat
+
+    var DKIM_SELECTOR = 'mail';
+    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
+
+    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
+    var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
+
+    if (publicKey === null) {
+        console.error('Error reading dkim public key. Stop DNS setup.');
+        return;
+    }
+
+    // remove header, footer and new lines
+    publicKey = publicKey.split('\n').slice(1, -2).join('');
+
+    // note that dmarc requires special DNS records for external RUF and RUA
+    var records = [
+        // naked domain
+        { subdomain: '', type: 'A', value: sysinfo.getIp() },
+        // webadmin domain
+        { subdomain: 'my', type: 'A', value: sysinfo.getIp() },
+        // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
+        { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
+        // t=s limits the domainkey to this domain and not it's subdomains
+        { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', value: '"v=DKIM1; t=s; p=' + publicKey + '"' },
+        // DMARC requires special setup if report email id is in different domain
+        { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
+    ];
+
+    debug('addDnsRecords:', records);
+
+    subdomains.addMany(records, function (error, changeIds) {
+        if (error) {
+            console.error('Admin DNS record addition failed', error);
+            gAddDnsRecordsTimerId = setTimeout(addDnsRecords, 10000);
+            return;
+        }
+
+        function checkIfInSync() {
+            debug('addDnsRecords: Check if admin DNS record is in sync.');
+
+            async.eachSeries(changeIds, function (changeId, callback) {
+                subdomains.status(changeId, function (error, result) {
+                    if (error) return callback(new Error('Failed to check if admin DNS record is in sync.', error));
+
+                    if (result !== 'done') return callback(new Error(changeId + ' is not in sync. result:' + result));
+
+                    callback(null);
+                });
+            }, function (error) {
+                if (error) {
+                    debug(error.message);
+                    gAddDnsRecordsTimerId = setTimeout(checkIfInSync, 5000);
+                    return;
+                }
+                debug('addDnsRecords: done');
+                config.setDnsInSync('DNS is in sync for ip ' + sysinfo.getIp());
+                sendHeartbeat(); // send heartbeat after the dns records are done
+            });
+        }
+
+        checkIfInSync();
+    });
+}
+
+function setCertificate(certificate, key, callback) {
+    assert.strictEqual(typeof certificate, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Updating certificates');
+
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), certificate)) {
+        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+    }
+
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) {
+        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+    }
+
+    shell.sudo('setCertificate', [ RELOAD_NGINX_CMD ], function (error) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function reboot(callback) {
+    shell.sudo('reboot', [ REBOOT_CMD ], callback);
+}
+
+function migrate(size, region, callback) {
+    assert.strictEqual(typeof size, 'string');
+    assert.strictEqual(typeof region, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_MIGRATE);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    function unlock(error) {
+        if (error) {
+            debug('Failed to migrate', error);
+            locker.unlock(locker.OP_MIGRATE);
+        } else {
+            debug('Migration initiated successfully');
+            // do not unlock; cloudron is migrating
+        }
+
+        return;
+    }
+
+    // initiate the migration in the background
+    backupBoxAndApps(function (error, restoreKey) {
+        if (error) return unlock(error);
+
+        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
+
+        superagent
+          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+          .query({ token: config.token() })
+          .send({ size: size, region: region, restoreKey: restoreKey })
+          .end(function (error, result) {
+            if (error) return unlock(error);
+            if (result.status === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+            if (result.status === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+            if (result.status !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return unlock(null);
+        });
+    });
+
+    callback(null);
+}
+
+function update(boxUpdateInfo, callback) {
+    assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!boxUpdateInfo) return callback(null);
+
+    var error = locker.lock(locker.OP_BOX_UPDATE);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    // initiate the update/upgrade but do not wait for it
+    if (boxUpdateInfo.upgrade) {
+        debug('Starting upgrade');
+        doUpgrade(boxUpdateInfo, function (error) {
+            if (error) {
+                debug('Upgrade failed with error: %s', error);
+                locker.unlock(locker.OP_BOX_UPDATE);
+            }
+        });
+    } else {
+        debug('Starting update');
+        doUpdate(boxUpdateInfo, function (error) {
+            if (error) {
+                debug('Update failed with error: %s', error);
+                locker.unlock(locker.OP_BOX_UPDATE);
+            }
+        });
+    }
+
+    callback(null);
+}
+
+function doUpgrade(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
+
+    function upgradeError(e) {
+        progress.set(progress.UPDATE, -1, e.message);
+        callback(e);
+    }
+
+    progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
+
+    backupBoxAndApps(function (error) {
+        if (error) return upgradeError(error);
+
+        superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
+          .query({ token: config.token() })
+          .send({ version: boxUpdateInfo.version })
+          .end(function (error, result) {
+            if (error) return upgradeError(new Error('Error making upgrade request: ' + error));
+            if (result.status !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
+
+            progress.set(progress.UPDATE, 10, 'Updating base system');
+
+            // no need to unlock since this is the last thing we ever do on this box
+
+            callback(null);
+        });
+    });
+}
+
+function doUpdate(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo && typeof boxUpdateInfo === 'object');
+
+    function updateError(e) {
+        progress.set(progress.UPDATE, -1, e.message);
+        callback(e);
+    }
+
+    progress.set(progress.UPDATE, 5, 'Backing up for update');
+
+    backupBoxAndApps(function (error) {
+        if (error) return updateError(error);
+
+        // fetch a signed sourceTarballUrl
+        superagent.get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/sourcetarballurl')
+          .query({ token: config.token(), boxVersion: boxUpdateInfo.version })
+          .end(function (error, result) {
+            if (error) return updateError(new Error('Error fetching sourceTarballUrl: ' + error));
+            if (result.status !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.status));
+            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + JSON.stringify(result.body)));
+
+            // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
+            var args = {
+                sourceTarballUrl: result.body.url,
+
+                // this data is opaque to the installer
+                data: {
+                    apiServerOrigin: config.apiServerOrigin(),
+                    aws: config.aws(),
+                    backupKey: config.backupKey(),
+                    boxVersionsUrl: config.get('boxVersionsUrl'),
+                    fqdn: config.fqdn(),
+                    isCustomDomain: config.isCustomDomain(),
+                    restoreUrl: null,
+                    restoreKey: null,
+                    token: config.token(),
+                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    version: boxUpdateInfo.version,
+                    webServerOrigin: config.webServerOrigin()
+                }
+            };
+
+            debug('updating box %j', args);
+
+            superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
+                if (error) return updateError(error);
+                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));
+
+                progress.set(progress.UPDATE, 10, 'Updating cloudron software');
+
+                callback(null);
+            });
+        });
+
+        // Do not add any code here. The installer script will stop the box code any instant
+    });
+}
+
+function backup(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    // clearing backup ensures tools can 'wait' on progress
+    progress.clear(progress.BACKUP);
+
+    // start the backup operation in the background
+    backupBoxAndApps(function (error) {
+        if (error) console.error('backup failed.', error);
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(callback) {
+    callback = callback || function () { };
+
+    backups.getAllPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        backup(callback);
+    });
+}
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    backups.getBackupUrl(null /* app */, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        debug('backup: url %s', result.url);
+
+        async.series([
+            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
+            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
+        ], function (error) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            debug('backup: successful');
+
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
+                callback(null, result.id);
+            });
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBox(callback) {
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+
+        backupBoxWithAppBackupIds(appBackupIds, callback);
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(callback) {
+    callback = callback || function () { }; // callback can be empty for timer triggered backup
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                if (error && error.reason !== AppsError.BAD_STATE) {
+                    debugApp(app, 'Unable to backup', error);
+                    return iteratorCallback(error);
+                }
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+                callback(error, restoreKey);
+            });
+        });
+    });
+}

src/collectd.config.ejs

@@ -0,0 +1,32 @@
+LoadPlugin "table"
+<Plugin table>
+  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.stat">
+    Instance "<%= appId %>-memory"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.max_usage_in_bytes">
+    Instance "<%= appId %>-memory"
+    Separator "\\n"
+    <Result>
+      Type gauge
+      InstancePrefix "max_usage_in_bytes"
+      ValuesFrom 0
+    </Result>
+  </Table>
+
+  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker-<%= containerId %>.scope/cpuacct.stat">
+    Instance "<%= appId %>-cpu"
+    Separator " \\n"
+    <Result>
+      Type gauge
+      InstancesFrom 0
+      ValuesFrom 1
+    </Result>
+  </Table>
+</Plugin>

src/config.js

@@ -0,0 +1,214 @@
+/* jslint node: true */
+
+'use strict';
+
+exports = module.exports = {
+    baseDir: baseDir,
+    dnsInSync: dnsInSync,
+    setDnsInSync: setDnsInSync,
+
+    // values set here will be lost after a upgrade/update. use the sqlite database
+    // for persistent values that need to be backed up
+    get: get,
+    set: set,
+
+    // ifdefs to check environment
+    CLOUDRON: process.env.BOX_ENV === 'cloudron',
+    TEST: process.env.BOX_ENV === 'test',
+
+    // convenience getters
+    apiServerOrigin: apiServerOrigin,
+    webServerOrigin: webServerOrigin,
+    fqdn: fqdn,
+    token: token,
+    version: version,
+    isCustomDomain: isCustomDomain,
+    database: database,
+
+    // these values are derived
+    adminOrigin: adminOrigin,
+    internalAdminOrigin: internalAdminOrigin,
+    appFqdn: appFqdn,
+    zoneName: zoneName,
+
+    isDev: isDev,
+
+    backupKey: backupKey,
+    aws: aws,
+
+    // for testing resets to defaults
+    _reset: initConfig
+};
+
+var assert = require('assert'),
+    constants = require('./constants.js'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
+var homeDir = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
+
+var data = { };
+
+function baseDir() {
+    if (exports.CLOUDRON) return homeDir;
+    if (exports.TEST) return path.join(homeDir, '.cloudron_test');
+}
+
+var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
+
+function dnsInSync() {
+    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
+}
+
+function setDnsInSync(content) {
+    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
+}
+
+function saveSync() {
+    fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
+}
+
+function initConfig() {
+    // setup defaults
+    data.fqdn = 'localhost';
+
+    data.token = null;
+    data.mailServer = null;
+    data.adminEmail = null;
+    data.mailDnsRecordIds = [ ];
+    data.boxVersionsUrl = null;
+    data.version = null;
+    data.isCustomDomain = false;
+    data.webServerOrigin = null;
+    data.internalPort = 3001;
+    data.ldapPort = 3002;
+    data.oauthProxyPort = 3003;
+    data.simpleAuthPort = 3004;
+    data.backupKey = 'backupKey';
+    data.aws = {
+        backupBucket: null,
+        backupPrefix: null,
+        accessKeyId: null,      // selfhosting only
+        secretAccessKey: null   // selfhosting only
+    };
+
+    if (exports.CLOUDRON) {
+        data.port = 3000;
+        data.apiServerOrigin = null;
+        data.database = null;
+    } else if (exports.TEST) {
+        data.port = 5454;
+        data.apiServerOrigin = 'http://localhost:6060'; // hock doesn't support https
+        data.database = {
+            hostname: 'localhost',
+            username: 'root',
+            password: '',
+            port: 3306,
+            name: 'boxtest'
+        };
+        data.token = 'APPSTORE_TOKEN';
+        data.aws.backupBucket = 'testbucket';
+        data.aws.backupPrefix = 'testprefix';
+        data.aws.endpoint = 'http://localhost:5353';
+    } else {
+        assert(false, 'Unknown environment. This should not happen!');
+    }
+
+    if (safe.fs.existsSync(cloudronConfigFileName)) {
+        var existingData = safe.JSON.parse(safe.fs.readFileSync(cloudronConfigFileName, 'utf8'));
+        _.extend(data, existingData); // overwrite defaults with saved config
+        return;
+    }
+
+    saveSync();
+}
+
+// cleanup any old config file we have for tests
+if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
+
+initConfig();
+
+// set(obj) or set(key, value)
+function set(key, value) {
+    if (typeof key === 'object') {
+        var obj = key;
+        for (var k in obj) {
+            assert(k in data, 'config.js is missing key "' + k + '"');
+            data[k] = obj[k];
+        }
+    } else {
+        data = safe.set(data, key, value);
+    }
+    saveSync();
+}
+
+function get(key) {
+    assert.strictEqual(typeof key, 'string');
+
+    return safe.query(data, key);
+}
+
+function apiServerOrigin() {
+    return get('apiServerOrigin');
+}
+
+function webServerOrigin() {
+    return get('webServerOrigin');
+}
+
+function fqdn() {
+    return get('fqdn');
+}
+
+// keep this in sync with start.sh admin.conf generation code
+function appFqdn(location) {
+    assert.strictEqual(typeof location, 'string');
+
+    if (location === '') return fqdn();
+    return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
+}
+
+function adminOrigin() {
+    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
+}
+
+function internalAdminOrigin() {
+    return 'http://127.0.0.1:' + get('port');
+}
+
+function token() {
+    return get('token');
+}
+
+function version() {
+    return get('version');
+}
+
+function isCustomDomain() {
+    return get('isCustomDomain');
+}
+
+function zoneName() {
+    if (isCustomDomain()) return fqdn(); // the appstore sets up the custom domain as a zone
+
+    // for shared domain name, strip out the hostname
+    return fqdn().substr(fqdn().indexOf('.') + 1);
+}
+
+function database() {
+    return get('database');
+}
+
+function isDev() {
+    return /dev/i.test(get('boxVersionsUrl'));
+}
+
+function backupKey() {
+    return get('backupKey');
+}
+
+function aws() {
+    return get('aws');
+}

src/constants.js

@@ -0,0 +1,12 @@
+'use strict';
+
+ // default admin installation location. keep in sync with ADMIN_LOCATION in setup/start.sh and BOX_ADMIN_LOCATION in appstore constants.js
+exports = module.exports = {
+    ADMIN_LOCATION: 'my',
+    API_LOCATION: 'api', // this is unused but reserved for future use (#403)
+    ADMIN_NAME: 'Settings',
+
+    ADMIN_CLIENT_ID: 'webadmin', // oauth client id
+    ADMIN_APPID: 'admin' // admin appid (settingsdb)
+};
+

src/cron.js

@@ -0,0 +1,181 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    cloudron = require('./cloudron.js'),
+    config = require('./config.js'),
+    CronJob = require('cron').CronJob,
+    debug = require('debug')('box:cron'),
+    janitor = require('./janitor.js'),
+    scheduler = require('./scheduler.js'),
+    settings = require('./settings.js'),
+    updateChecker = require('./updatechecker.js');
+
+var gAutoupdaterJob = null,
+    gBoxUpdateCheckerJob = null,
+    gAppUpdateCheckerJob = null,
+    gHeartbeatJob = null,
+    gBackupJob = null,
+    gCleanupTokensJob = null,
+    gDockerVolumeCleanerJob = null,
+    gSchedulerSyncJob = null;
+
+var gInitialized = false;
+
+var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+
+// cron format
+// Seconds: 0-59
+// Minutes: 0-59
+// Hours: 0-23
+// Day of Month: 1-31
+// Months: 0-11
+// Day of Week: 0-6
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gInitialized) return callback();
+
+    settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
+    settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
+
+    gInitialized = true;
+
+    recreateJobs(callback);
+}
+
+function recreateJobs(unusedTimeZone, callback) {
+    if (typeof unusedTimeZone === 'function') callback = unusedTimeZone;
+
+    settings.getAll(function (error, allSettings) {
+        debug('Creating jobs with timezone %s', allSettings[settings.TIME_ZONE_KEY]);
+
+        if (gHeartbeatJob) gHeartbeatJob.stop();
+        gHeartbeatJob = new CronJob({
+            cronTime: '00 */1 * * * *', // every minute
+            onTick: cloudron.sendHeartbeat,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gBackupJob) gBackupJob.stop();
+        gBackupJob = new CronJob({
+            cronTime: '00 00 */4 * * *', // every 4 hours
+            onTick: cloudron.ensureBackup,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
+        gBoxUpdateCheckerJob = new CronJob({
+            cronTime: '00 */10 * * * *', // every 10 minutes
+            onTick: updateChecker.checkBoxUpdates,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
+        gAppUpdateCheckerJob = new CronJob({
+            cronTime: '00 */10 * * * *', // every 10 minutes
+            onTick: updateChecker.checkAppUpdates,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gCleanupTokensJob) gCleanupTokensJob.stop();
+        gCleanupTokensJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: janitor.cleanupTokens,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
+        gDockerVolumeCleanerJob = new CronJob({
+            cronTime: '00 00 */12 * * *', // every 12 hours
+            onTick: janitor.cleanupDockerVolumes,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
+        gSchedulerSyncJob = new CronJob({
+            cronTime: config.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
+            onTick: scheduler.sync,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY]);
+
+        if (callback) callback();
+    });
+}
+
+function autoupdatePatternChanged(pattern) {
+    assert.strictEqual(typeof pattern, 'string');
+
+    debug('Auto update pattern changed to %s', pattern);
+
+    if (gAutoupdaterJob) gAutoupdaterJob.stop();
+
+    if (pattern === 'never') return;
+
+    gAutoupdaterJob = new CronJob({
+        cronTime: pattern,
+        onTick: function() {
+            var updateInfo = updateChecker.getUpdateInfo();
+            if (updateInfo.box) {
+                debug('Starting autoupdate to %j', updateInfo.box);
+                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+            } else if (updateInfo.apps) {
+                debug('Starting app update to %j', updateInfo.apps);
+                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+            } else {
+                debug('No auto updates available');
+            }
+        },
+        start: true,
+        timeZone: gBoxUpdateCheckerJob.cronTime.zone // hack
+    });
+}
+
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!gInitialized) return callback();
+
+    if (gAutoupdaterJob) gAutoupdaterJob.stop();
+    gAutoupdaterJob = null;
+
+    gBoxUpdateCheckerJob.stop();
+    gBoxUpdateCheckerJob = null;
+
+    gAppUpdateCheckerJob.stop();
+    gAppUpdateCheckerJob = null;
+
+    gHeartbeatJob.stop();
+    gHeartbeatJob = null;
+
+    gBackupJob.stop();
+    gBackupJob = null;
+
+    gCleanupTokensJob.stop();
+    gCleanupTokensJob = null;
+
+    gDockerVolumeCleanerJob.stop();
+    gDockerVolumeCleanerJob = null;
+
+    gSchedulerSyncJob.stop();
+    gSchedulerSyncJob = null;
+
+    gInitialized = false;
+
+    callback();
+}

src/database.js

@@ -0,0 +1,202 @@
+/* jslint node: true */
+
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize,
+    query: query,
+    transaction: transaction,
+
+    beginTransaction: beginTransaction,
+    rollback: rollback,
+    commit: commit,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    once = require('once'),
+    config = require('./config.js'),
+    mysql = require('mysql'),
+    util = require('util');
+
+var gConnectionPool = null,
+    gDefaultConnection = null;
+
+function initialize(options, callback) {
+    if (typeof options === 'function') {
+        callback = options;
+        options = {
+            connectionLimit: 5
+        };
+    }
+
+    assert.strictEqual(typeof options.connectionLimit, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gConnectionPool !== null) return callback(null);
+
+    gConnectionPool  = mysql.createPool({
+        connectionLimit: options.connectionLimit,
+        host: config.database().hostname,
+        user: config.database().username,
+        password: config.database().password,
+        port: config.database().port,
+        database: config.database().name,
+        multipleStatements: false,
+        ssl: false
+    });
+
+    reconnect(callback);
+}
+
+function uninitialize(callback) {
+    if (gConnectionPool) {
+        gConnectionPool.end(callback);
+        gConnectionPool = null;
+    } else {
+        callback(null);
+    }
+}
+
+function setupConnection(connection, callback) {
+    assert.strictEqual(typeof connection, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.on('error', console.error);
+
+    async.series([
+        connection.query.bind(connection, 'USE ' + config.database().name),
+        connection.query.bind(connection, 'SET SESSION sql_mode = \'strict_all_tables\'')
+    ], function (error) {
+        connection.removeListener('error', console.error);
+
+        if (error) connection.release();
+
+        callback(error);
+    });
+}
+
+function reconnect(callback) {
+    callback = callback ? once(callback) : function () {};
+
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) {
+            console.error('Unable to reestablish connection to database. Try again in a bit.', error.message);
+            return setTimeout(reconnect.bind(null, callback), 1000);
+        }
+
+        connection.on('error', function (error) {
+            // by design, we catch all normal errors by providing callbacks.
+            // this function should be invoked only when we have no callbacks pending and we have a fatal error
+            assert(error.fatal, 'Non-fatal error on connection object');
+
+            console.error('Unhandled mysql connection error.', error);
+
+            // This is most likely an issue an can cause double callbacks from reconnect()
+            setTimeout(reconnect.bind(null, callback), 1000);
+        });
+
+        setupConnection(connection, function (error) {
+            if (error) return setTimeout(reconnect.bind(null, callback), 1000);
+
+            gDefaultConnection = connection;
+
+            callback(null);
+        });
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // the clear funcs don't completely clear the db, they leave the migration code defaults
+    async.series([
+        require('./appdb.js')._clear,
+        require('./authcodedb.js')._clear,
+        require('./clientdb.js')._clear,
+        require('./tokendb.js')._clear,
+        require('./userdb.js')._clear,
+        require('./settingsdb.js')._clear
+    ], callback);
+}
+
+function beginTransaction(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) return callback(error);
+
+        setupConnection(connection, function (error) {
+            if (error) return callback(error);
+
+            connection.beginTransaction(function (error) {
+                if (error) return callback(error);
+
+                return callback(null, connection);
+            });
+        });
+    });
+}
+
+function rollback(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.rollback(function (error) {
+        if (error) console.error(error); // can this happen?
+
+        connection.release();
+        callback(null);
+    });
+}
+
+// FIXME: if commit fails, is it supposed to return an error ?
+function commit(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.commit(function (error) {
+        if (error) return rollback(connection, callback);
+
+        connection.release();
+        return callback(null);
+    });
+}
+
+function query() {
+    var args = Array.prototype.slice.call(arguments);
+    var callback = args[args.length - 1];
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
+
+    args[args.length -1 ] = function (error, result) {
+        if (error && error.fatal) {
+            gDefaultConnection = null;
+            setTimeout(reconnect, 1000);
+        }
+
+        callback(error, result);
+    };
+
+    gDefaultConnection.query.apply(gDefaultConnection, args);
+}
+
+function transaction(queries, callback) {
+    assert(util.isArray(queries));
+    assert.strictEqual(typeof callback, 'function');
+
+    beginTransaction(function (error, conn) {
+        if (error) return callback(error);
+
+        async.mapSeries(queries, function iterator(query, done) {
+            conn.query(query.query, query.args, done);
+        }, function seriesDone(error, results) {
+            if (error) return rollback(conn, callback.bind(null, error));
+
+            commit(conn, callback.bind(null, null, results));
+        });
+    });
+}
+

src/databaseerror.js

@@ -0,0 +1,32 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = DatabaseError;
+
+var assert = require('assert'),
+    util = require('util');
+
+function DatabaseError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined' || errorOrMessage === null);
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined' || errorOrMessage === null) {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(DatabaseError, Error);
+
+DatabaseError.INTERNAL_ERROR = 'Internal error';
+DatabaseError.ALREADY_EXISTS = 'Entry already exist';
+DatabaseError.NOT_FOUND = 'Record not found';
+DatabaseError.BAD_FIELD = 'Invalid field';

src/developer.js

@@ -0,0 +1,85 @@
+/* jslint node: true */
+
+'use strict';
+
+exports = module.exports = {
+    DeveloperError: DeveloperError,
+
+    enabled: enabled,
+    setEnabled: setEnabled,
+    issueDeveloperToken: issueDeveloperToken,
+    getNonApprovedApps: getNonApprovedApps
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    tokendb = require('./tokendb.js'),
+    settings = require('./settings.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function DeveloperError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(DeveloperError, Error);
+DeveloperError.INTERNAL_ERROR = 'Internal Error';
+
+function enabled(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDeveloperMode(function (error, enabled) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        callback(null, enabled);
+    });
+}
+
+function setEnabled(enabled, callback) {
+    assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.setDeveloperMode(enabled, function (error) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function issueDeveloperToken(user, callback) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var token = tokendb.generateToken();
+    var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+
+    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        callback(null, { token: token, expiresAt: expiresAt });
+    });
+}
+
+function getNonApprovedApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
+    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        if (result.status !== 200) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+
+        callback(null, result.body.apps || []);
+    });
+}

src/digitalocean.js

@@ -0,0 +1,46 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    checkPtrRecord: checkPtrRecord
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:digitalocean'),
+    dns = require('native-dns');
+
+function checkPtrRecord(ip, fqdn, callback) {
+    assert(ip === null || typeof ip === 'string');
+    assert.strictEqual(typeof fqdn, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('checkPtrRecord: ' + ip);
+
+    if (!ip) return callback(new Error('Network down'));
+
+    dns.resolve4('ns1.digitalocean.com', function (error, rdnsIps) {
+        if (error || rdnsIps.length === 0) return callback(new Error('Failed to query DO DNS'));
+
+        var reversedIp = ip.split('.').reverse().join('.');
+
+        var req = dns.Request({
+            question: dns.Question({ name: reversedIp + '.in-addr.arpa', type: 'PTR' }),
+            server: { address: rdnsIps[0] },
+            timeout: 5000
+        });
+
+        req.on('timeout', function () { return callback(new Error('Timedout')); });
+
+        req.on('message', function (error, message) {
+            if (error || !message.answer || message.answer.length === 0) return callback(new Error('Failed to query PTR'));
+
+            debug('checkPtrRecord: Actual:%s Expecting:%s', message.answer[0].data, fqdn);
+            callback(null, message.answer[0].data === fqdn);
+        });
+
+        req.send();
+    });
+}
+
+

src/docker.js

@@ -0,0 +1,343 @@
+'use strict';
+
+var addons = require('./addons.js'),
+    async = require('async'),
+    assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:src/docker.js'),
+    Docker = require('dockerode'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    util = require('util');
+
+exports = module.exports = {
+    connection: connectionInstance(),
+    downloadImage: downloadImage,
+    createContainer: createContainer,
+    startContainer: startContainer,
+    stopContainer: stopContainer,
+    stopContainers: stopContainers,
+    deleteContainer: deleteContainer,
+    deleteImage: deleteImage,
+    deleteContainers: deleteContainers,
+    createSubcontainer: createSubcontainer
+};
+
+function connectionInstance() {
+    var docker;
+
+    if (process.env.BOX_ENV === 'test') {
+        // test code runs a docker proxy on this port
+        docker = new Docker({ host: 'http://localhost', port: 5687 });
+
+        // proxy code uses this to route to the real docker
+        docker.options = { socketPath: '/var/run/docker.sock' };
+    } else {
+        docker = new Docker({ socketPath: '/var/run/docker.sock' });
+    }
+
+    return docker;
+}
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? (app.location || '(bare)') : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function targetBoxVersion(manifest) {
+    if ('targetBoxVersion' in manifest) return manifest.targetBoxVersion;
+
+    if ('minBoxVersion' in manifest) return manifest.minBoxVersion;
+
+    return '0.0.1';
+}
+
+function pullImage(manifest, callback) {
+    var docker = exports.connection;
+
+    docker.pull(manifest.dockerImage, function (err, stream) {
+        if (err) return callback(new Error('Error connecting to docker. statusCode: %s' + err.statusCode));
+
+        // https://github.com/dotcloud/docker/issues/1074 says each status message
+        // is emitted as a chunk
+        stream.on('data', function (chunk) {
+            var data = safe.JSON.parse(chunk) || { };
+            debug('pullImage data: %j', data);
+
+            // The information here is useless because this is per layer as opposed to per image
+            if (data.status) {
+                // debugApp(app, 'progress: %s', data.status); // progressDetail { current, total }
+            } else if (data.error) {
+                debug('pullImage error detail: %s', data.errorDetail.message);
+            }
+        });
+
+        stream.on('end', function () {
+            debug('downloaded image %s successfully', manifest.dockerImage);
+
+            var image = docker.getImage(manifest.dockerImage);
+
+            image.inspect(function (err, data) {
+                if (err) return callback(new Error('Error inspecting image:' + err.message));
+                if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
+                if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
+
+                debug('This image exposes ports: %j', data.Config.ExposedPorts);
+
+                callback(null);
+            });
+        });
+
+        stream.on('error', function (error) {
+            debug('error pulling image %s : %j', manifest.dockerImage, error);
+
+            callback(error);
+        });
+    });
+}
+
+function downloadImage(manifest, callback) {
+    assert.strictEqual(typeof manifest, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('downloadImage %s', manifest.dockerImage);
+
+    var attempt = 1;
+
+    async.retry({ times: 5, interval: 15000 }, function (retryCallback) {
+        debug('Downloading image. attempt: %s', attempt++);
+
+        pullImage(manifest, function (error) {
+            if (error) console.error(error);
+
+            retryCallback(error);
+        });
+    }, callback);
+}
+
+function createSubcontainer(app, cmd, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!cmd || util.isArray(cmd));
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection,
+        isAppContainer = !cmd;
+
+    var manifest = app.manifest;
+    var exposedPorts = {}, dockerPortBindings = { };
+    var stdEnv = [
+        'CLOUDRON=1',
+        'WEBADMIN_ORIGIN=' + config.adminOrigin(),
+        'API_ORIGIN=' + config.adminOrigin(),
+        'APP_ORIGIN=https://' + config.appFqdn(app.location),
+        'APP_DOMAIN=' + config.appFqdn(app.location)
+    ];
+
+    // docker portBindings requires ports to be exposed
+    exposedPorts[manifest.httpPort + '/tcp'] = {};
+
+    dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: '127.0.0.1', HostPort: app.httpPort + '' } ];
+
+    var portEnv = [];
+    for (var e in app.portBindings) {
+        var hostPort = app.portBindings[e];
+        var containerPort = manifest.tcpPorts[e].containerPort || hostPort;
+
+        exposedPorts[containerPort + '/tcp'] = {};
+        portEnv.push(e + '=' + hostPort);
+
+        dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
+    }
+
+    var memoryLimit = manifest.memoryLimit || 1024 * 1024 * 200; // 200mb by default
+
+    addons.getEnvironment(app, function (error, addonEnv) {
+        if (error) return callback(new Error('Error getting addon environment : ' + error));
+
+        var containerOptions = {
+            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
+            Hostname: semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location),
+            Tty: isAppContainer,
+            Image: app.manifest.dockerImage,
+            Cmd: cmd,
+            Env: stdEnv.concat(addonEnv).concat(portEnv),
+            ExposedPorts: isAppContainer ? exposedPorts : { },
+            Volumes: { // see also ReadonlyRootfs
+                '/tmp': {},
+                '/run': {}
+            },
+            Labels: {
+                "location": app.location,
+                "appId": app.id,
+                "isSubcontainer": String(!isAppContainer)
+            },
+            HostConfig: {
+                Binds: addons.getBindsSync(app, app.manifest.addons),
+                Memory: memoryLimit / 2,
+                MemorySwap: memoryLimit, // Memory + Swap
+                PortBindings: isAppContainer ? dockerPortBindings : { },
+                PublishAllPorts: false,
+                ReadonlyRootfs: semver.gte(targetBoxVersion(app.manifest), '0.0.66'), // see also Volumes in startContainer
+                Links: addons.getLinksSync(app, app.manifest.addons),
+                RestartPolicy: {
+                    "Name": isAppContainer ? "always" : "no",
+                    "MaximumRetryCount": 0
+                },
+                CpuShares: 512, // relative to 1024 for system processes
+                VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
+                SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
+            }
+        };
+
+        // older versions wanted a writable /var/log
+        if (semver.lte(targetBoxVersion(app.manifest), '0.0.71')) containerOptions.Volumes['/var/log'] = {};
+
+        debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
+
+        docker.createContainer(containerOptions, callback);
+    });
+}
+
+function createContainer(app, callback) {
+    createSubcontainer(app, null, callback);
+}
+
+function startContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    var container = docker.getContainer(containerId);
+    debug('Starting container %s', containerId);
+
+    container.start(function (error) {
+        if (error && error.statusCode !== 304) return callback(new Error('Error starting container :' + error));
+
+        return callback(null);
+    });
+}
+
+function stopContainer(containerId, callback) {
+    assert(!containerId || typeof containerId === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!containerId) {
+        debug('No previous container to stop');
+        return callback();
+    }
+
+    var docker = exports.connection;
+    var container = docker.getContainer(containerId);
+    debug('Stopping container %s', containerId);
+
+    var options = {
+        t: 10 // wait for 10 seconds before killing it
+    };
+
+    container.stop(options, function (error) {
+        if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error stopping container:' + error));
+
+        debug('Waiting for container ' + containerId);
+
+        container.wait(function (error, data) {
+            if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error waiting on container:' + error));
+
+            debug('Container %s stopped with status code [%s]', containerId, data ? String(data.StatusCode) : '');
+
+            return callback(null);
+        });
+    });
+}
+
+function deleteContainer(containerId, callback) {
+    assert(!containerId || typeof containerId === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('deleting container %s', containerId);
+
+    if (containerId === null) return callback(null);
+
+    var docker = exports.connection;
+    var container = docker.getContainer(containerId);
+
+    var removeOptions = {
+        force: true, // kill container if it's running
+        v: true // removes volumes associated with the container (but not host mounts)
+    };
+
+    container.remove(removeOptions, function (error) {
+        if (error && error.statusCode === 404) return callback(null);
+
+        if (error) debug('Error removing container %s : %j', containerId, error);
+
+        callback(error);
+    });
+}
+
+function deleteContainers(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    debug('deleting containers of %s', appId);
+
+    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            deleteContainer(container.Id, iteratorDone);
+        }, callback);
+    });
+}
+
+function stopContainers(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    debug('stopping containers of %s', appId);
+
+    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            stopContainer(container.Id, iteratorDone);
+        }, callback);
+    });
+}
+
+function deleteImage(manifest, callback) {
+    assert(!manifest || typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var dockerImage = manifest ? manifest.dockerImage : null;
+    if (!dockerImage) return callback(null);
+
+    var docker = exports.connection;
+
+    docker.getImage(dockerImage).inspect(function (error, result) {
+        if (error && error.statusCode === 404) return callback(null);
+
+        if (error) return callback(error);
+
+        var removeOptions = {
+            force: true,
+            noprune: false
+        };
+
+         // delete image by id because 'docker pull' pulls down all the tags and this is the only way to delete all tags
+        docker.getImage(result.Id).remove(removeOptions, function (error) {
+            if (error && error.statusCode === 404) return callback(null);
+            if (error && error.statusCode === 409) return callback(null); // another container using the image
+
+            if (error) debug('Error removing image %s : %j', dockerImage, error);
+
+            callback(error);
+        });
+    });
+}

src/janitor.js

@@ -0,0 +1,103 @@
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    authcodedb = require('./authcodedb.js'),
+    debug = require('debug')('box:src/janitor'),
+    docker = require('./docker.js').connection,
+    tokendb = require('./tokendb.js');
+
+exports = module.exports = {
+    cleanupTokens: cleanupTokens,
+    cleanupDockerVolumes: cleanupDockerVolumes
+};
+
+var NOOP_CALLBACK = function () { };
+
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+
+            callback();
+        });
+    };
+}
+
+function cleanupExpiredTokens(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired tokens.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupExpiredAuthCodes(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    authcodedb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired authcodes.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupTokens(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    debug('Cleaning up expired tokens');
+
+    async.series([
+        ignoreError(cleanupExpiredTokens),
+        ignoreError(cleanupExpiredAuthCodes)
+    ], callback);
+}
+
+function cleanupTmpVolume(containerInfo, callback) {
+    assert.strictEqual(typeof containerInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var cmd = 'find /tmp -mtime +10 -exec rm -rf {} +'.split(' '); // 10 days old
+
+    debug('cleanupTmpVolume %j', containerInfo.Names);
+
+    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+
+        execContainer.start(function(err, stream) {
+            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+
+            stream.on('error', callback);
+            stream.on('end', callback);
+
+            stream.setEncoding('utf8');
+            stream.pipe(process.stdout);
+        });
+    });
+}
+
+function cleanupDockerVolumes(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Cleaning up docker volumes');
+
+    docker.listContainers({ all: 0 }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            cleanupTmpVolume(container, function (error) {
+                if (error) debug('Error cleaning tmp: %s', error);
+
+                iteratorDone(); // intentionally ignore error
+            });
+        }, callback);
+    });
+}

src/ldap.js

@@ -0,0 +1,138 @@
+'use strict';
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:ldap'),
+    user = require('./user.js'),
+    UserError = user.UserError,
+    ldap = require('ldapjs');
+
+var gServer = null;
+
+var NOOP = function () {};
+
+var gLogger = {
+    trace: NOOP,
+    debug: NOOP,
+    info: debug,
+    warn: debug,
+    error: console.error,
+    fatal: console.error
+};
+
+var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gServer = ldap.createServer({ log: gLogger });
+
+    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
+        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+
+        user.list(function (error, result){
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // send user objects
+            result.forEach(function (entry) {
+                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+                var groups = [ GROUP_USERS_DN ];
+                if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+                var tmp = {
+                    dn: dn.toString(),
+                    attributes: {
+                        objectclass: ['user'],
+                        objectcategory: 'person',
+                        cn: entry.id,
+                        uid: entry.id,
+                        mail: entry.email,
+                        displayname: entry.username,
+                        username: entry.username,
+                        samaccountname: entry.username,      // to support ActiveDirectory clients
+                        memberof: groups
+                    }
+                };
+
+                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
+                    res.send(tmp);
+                }
+            });
+
+            res.end();
+        });
+    });
+
+    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
+        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+
+        user.list(function (error, result){
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            var groups = [{
+                name: 'users',
+                admin: false
+            }, {
+                name: 'admins',
+                admin: true
+            }];
+
+            groups.forEach(function (group) {
+                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+                var tmp = {
+                    dn: dn.toString(),
+                    attributes: {
+                        objectclass: ['group'],
+                        cn: group.name,
+                        memberuid: members.map(function(entry) { return entry.id; })
+                    }
+                };
+
+                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
+                    res.send(tmp);
+                }
+            });
+
+            res.end();
+        });
+    });
+
+    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
+        // TODO: validate password
+        debug('ldap application bind: %s', req.dn.toString());
+        res.end();
+    });
+
+    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
+        debug('ldap user bind: %s', req.dn.toString());
+
+        if (!req.dn.rdns[0].cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+        user.verify(req.dn.rdns[0].cn, req.credentials || '', function (error, result) {
+            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+            if (error) return next(new ldap.OperationsError(error));
+
+            res.end();
+        });
+    });
+
+    gServer.listen(config.get('ldapPort'), callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gServer.close();
+
+    callback();
+}

src/locker.js

@@ -0,0 +1,71 @@
+'use strict';
+
+var assert = require('assert'),
+    debug = require('debug')('box:locker'),
+    EventEmitter = require('events').EventEmitter,
+    util = require('util');
+
+function Locker() {
+    this._operation = null;
+    this._timestamp = null;
+    this._watcherId = -1;
+    this._lockDepth = 0; // recursive locks
+}
+util.inherits(Locker, EventEmitter);
+
+// these are mutually exclusive operations
+Locker.prototype.OP_BOX_UPDATE = 'box_update';
+Locker.prototype.OP_FULL_BACKUP = 'full_backup';
+Locker.prototype.OP_APPTASK = 'apptask';
+Locker.prototype.OP_MIGRATE = 'migrate';
+
+Locker.prototype.lock = function (operation) {
+    assert.strictEqual(typeof operation, 'string');
+
+    if (this._operation !== null) return new Error('Already locked for ' + this._operation);
+
+    this._operation = operation;
+    ++this._lockDepth;
+    this._timestamp = new Date();
+    var that = this;
+    this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
+
+    debug('Acquired : %s', this._operation);
+
+    this.emit('locked', this._operation);
+
+    return null;
+};
+
+Locker.prototype.recursiveLock = function (operation) {
+    if (this._operation === operation) {
+        ++this._lockDepth;
+        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
+        return null;
+    }
+
+    return this.lock(operation);
+};
+
+Locker.prototype.unlock = function (operation) {
+    assert.strictEqual(typeof operation, 'string');
+
+    if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+
+    if (--this._lockDepth === 0) {
+        debug('Released : %s', this._operation);
+
+        this._operation = null;
+        this._timestamp = null;
+        clearInterval(this._watcherId);
+        this._watcherId = -1;
+    } else {
+        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
+    }
+
+    this.emit('unlocked', operation);
+
+    return null;
+};
+
+exports = module.exports = new Locker();

src/mail_templates/app_down.ejs

@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+The application titled '<%= title %>' that you installed at <%= appFqdn %>
+is not responding.
+
+This is most likely a problem in the application. Please report this issue to
+support@cloudron.io (by forwarding this email).
+
+You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
+
+Thank you,
+Application WatchDog
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/app_update_available.ejs

@@ -0,0 +1,15 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+A new version of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+
+Please update at your convenience at <%= webadminUrl %>.
+
+Thank you,
+Update Manager
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/box_update_available.ejs

@@ -0,0 +1,20 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+A new version of Cloudron <%= fqdn %> is available!
+
+Please update at your convenience at <%= webadminUrl %>.
+
+Changelog:
+<% for (var i = 0; i < changelog.length; i++) { %>
+  * <%- changelog[i] %>
+<% } %>
+
+Thank you,
+Update Manager
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/crash_notification.ejs

@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+
+Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
+
+Please see some excerpt of the logs below.
+
+Thank you,
+Your Cloudron
+
+-------------------------------------
+
+<%- context %>
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/feedback.ejs

@@ -0,0 +1,15 @@
+<%if (format === 'text') { %>
+
+New <%= type %> from <%= fqdn %>.
+
+Sender:  <%= user.email %>
+Sent at: <%= new Date().toUTCString() %>
+
+Subject: <%= subject %>
+-----------------------------------------------------------
+<%= description %>
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/password_reset.ejs

@@ -0,0 +1,20 @@
+<%if (format === 'text') { %>
+
+Dear <%= username %>,
+
+Someone, hopefully you, has requested your <%= fqdn %>'s account password
+be reset. If you did not request this reset, please ignore this message.
+
+To reset your password, please visit the following page:
+<%= resetLink %>
+
+When you visit the above page, you will be prompted to enter a new password.
+After you have submitted the form, you can login using the new password.
+
+Thank you,
+<%= fqdn %> Admin
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/user_event.ejs

@@ -0,0 +1,15 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+User with name '<%= username %>' (<%= email %>) <%= event %> in the Cloudron at <%= fqdn %>.
+
+You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
+
+Thank you,
+User Manager
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/welcome_user.ejs

@@ -0,0 +1,24 @@
+<%if (format === 'text') { %>
+
+Dear <%= user.username %>,
+
+I am excited to welcome you to my Cloudron <%= fqdn %>!
+
+The Cloudron is our own Private Cloud. You can read more about it
+at https://www.cloudron.io.
+
+           You username is '<%= user.username %>'
+
+To get started, create your account by visiting the following page:
+<%= setupLink %>
+
+When you visit the above page, you will be prompted to enter a new password.
+After you have submitted the form, you can login using the new password.
+
+Thank you,
+<%= invitor.email %>
+
+<% } else { %>
+
+<% } %>
+

src/mailer.js

@@ -0,0 +1,302 @@
+/* jslint node: true */
+
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize,
+
+    userAdded: userAdded,
+    userRemoved: userRemoved,
+    adminChanged: adminChanged,
+    passwordReset: passwordReset,
+    boxUpdateAvailable: boxUpdateAvailable,
+    appUpdateAvailable: appUpdateAvailable,
+
+    sendCrashNotification: sendCrashNotification,
+
+    appDied: appDied,
+
+    FEEDBACK_TYPE_FEEDBACK: 'feedback',
+    FEEDBACK_TYPE_TICKET: 'ticket',
+    FEEDBACK_TYPE_APP: 'app',
+    sendFeedback: sendFeedback
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    debug = require('debug')('box:mailer'),
+    digitalocean = require('./digitalocean.js'),
+    docker = require('./docker.js').connection,
+    ejs = require('ejs'),
+    nodemailer = require('nodemailer'),
+    path = require('path'),
+    safe = require('safetydance'),
+    smtpTransport = require('nodemailer-smtp-transport'),
+    sysinfo = require('./sysinfo.js'),
+    userdb = require('./userdb.js'),
+    util = require('util'),
+    _ = require('underscore');
+
+var MAIL_TEMPLATES_DIR = path.join(__dirname, 'mail_templates');
+
+var gMailQueue = [ ],
+    gDnsReady = false,
+    gCheckDnsTimerId = null;
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    checkDns();
+    callback(null);
+}
+
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // TODO: interrupt processQueue as well
+    clearTimeout(gCheckDnsTimerId);
+    gCheckDnsTimerId = null;
+
+    debug(gMailQueue.length + ' mail items dropped');
+    gMailQueue = [ ];
+
+    callback(null);
+}
+
+function checkDns() {
+    digitalocean.checkPtrRecord(sysinfo.getIp(), config.fqdn(), function (error, ok) {
+        if (error || !ok) {
+            debug('PTR record not setup yet');
+            gCheckDnsTimerId = setTimeout(checkDns, 10000);
+            return;
+        }
+
+        gDnsReady = true;
+        processQueue();
+    });
+}
+
+function processQueue() {
+    docker.getContainer('mail').inspect(function (error, data) {
+        if (error) return console.error(error);
+
+        var mailServerIp = safe.query(data, 'NetworkSettings.IPAddress');
+        if (!mailServerIp) return debug('Error querying mail server IP');
+
+        var transport = nodemailer.createTransport(smtpTransport({
+            host: mailServerIp,
+            port: 2500 // this value comes from mail container
+        }));
+
+        var mailQueueCopy = gMailQueue;
+        gMailQueue = [ ];
+
+        debug('Processing mail queue of size %d (through %s:2500)', mailQueueCopy.length, mailServerIp);
+
+        async.mapSeries(mailQueueCopy, function iterator(mailOptions, callback) {
+            transport.sendMail(mailOptions, function (error) {
+                if (error) return console.error(error); // TODO: requeue?
+                debug('Email sent to ' + mailOptions.to);
+            });
+            callback(null);
+        }, function done() {
+            debug('Done processing mail queue');
+        });
+    });
+}
+
+function enqueue(mailOptions) {
+    assert.strictEqual(typeof mailOptions, 'object');
+
+    debug('Queued mail for ' + mailOptions.from + ' to ' + mailOptions.to);
+    gMailQueue.push(mailOptions);
+
+    if (gDnsReady) processQueue();
+}
+
+function render(templateFile, params) {
+    assert.strictEqual(typeof templateFile, 'string');
+    assert.strictEqual(typeof params, 'object');
+
+    return ejs.render(safe.fs.readFileSync(path.join(MAIL_TEMPLATES_DIR, templateFile), 'utf8'), params);
+}
+
+function getAdminEmails(callback) {
+    userdb.getAllAdmins(function (error, admins) {
+        if (error) return callback(error);
+
+        var adminEmails = [ ];
+        admins.forEach(function (admin) { adminEmails.push(admin.email); });
+
+        callback(null, adminEmails);
+    });
+}
+
+function mailUserEventToAdmins(user, event) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof event, 'string');
+
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
+
+        adminEmails = _.difference(adminEmails, [ user.email ]);
+
+        var mailOptions = {
+            from: config.get('adminEmail'),
+            to: adminEmails.join(', '),
+            subject: util.format('%s %s in Cloudron %s', user.username, event, config.fqdn()),
+            text: render('user_event.ejs', { fqdn: config.fqdn(), username: user.username, email: user.email, event: event, format: 'text' }),
+        };
+
+        enqueue(mailOptions);
+    });
+}
+
+function userAdded(user, invitor) {
+    assert.strictEqual(typeof user, 'object');
+    assert(typeof invitor === 'object');
+
+    debug('Sending mail for userAdded');
+
+    var templateData = {
+        user: user,
+        webadminUrl: config.adminOrigin(),
+        setupLink: config.adminOrigin() + '/api/v1/session/password/setup.html?reset_token=' + user.resetToken,
+        format: 'text',
+        fqdn: config.fqdn(),
+        invitor: invitor
+    };
+
+    var mailOptions = {
+        from: config.get('adminEmail'),
+        to: user.email,
+        subject: util.format('Welcome to Cloudron %s', config.fqdn()),
+        text: render('welcome_user.ejs', templateData)
+    };
+
+    enqueue(mailOptions);
+
+    mailUserEventToAdmins(user, 'was added');
+}
+
+function userRemoved(username) {
+    assert.strictEqual(typeof username, 'string');
+
+    debug('Sending mail for userRemoved');
+
+    mailUserEventToAdmins({ username: username }, 'was removed');
+}
+
+function adminChanged(user) {
+    assert.strictEqual(typeof user, 'object');
+
+    debug('Sending mail for adminChanged');
+
+    mailUserEventToAdmins(user, user.admin ? 'is now an admin' : 'is no more an admin');
+}
+
+function passwordReset(user) {
+    assert.strictEqual(typeof user, 'object');
+
+    debug('Sending mail for password reset for user %s.', user.username);
+
+    var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
+
+    var mailOptions = {
+        from: config.get('adminEmail'),
+        to: user.email,
+        subject: 'Password Reset Request',
+        text: render('password_reset.ejs', { fqdn: config.fqdn(), username: user.username, resetLink: resetLink, format: 'text' })
+    };
+
+    enqueue(mailOptions);
+}
+
+function appDied(app) {
+    assert.strictEqual(typeof app, 'object');
+
+    debug('Sending mail for app %s @ %s died', app.id, app.location);
+
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
+
+        var mailOptions = {
+            from: config.get('adminEmail'),
+            to: adminEmails.join(', '),
+            subject: util.format('App %s is down', app.location),
+            text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
+        };
+
+        enqueue(mailOptions);
+    });
+}
+
+function boxUpdateAvailable(newBoxVersion, changelog) {
+    assert.strictEqual(typeof newBoxVersion, 'string');
+    assert(util.isArray(changelog));
+
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
+
+         var mailOptions = {
+            from: config.get('adminEmail'),
+            to: adminEmails.join(', '),
+            subject: util.format('%s has a new update available', config.fqdn()),
+            text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
+        };
+
+        enqueue(mailOptions);
+    });
+}
+
+function appUpdateAvailable(app, updateInfo) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof updateInfo, 'object');
+
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return console.log('Error getting admins', error);
+
+         var mailOptions = {
+            from: config.get('adminEmail'),
+            to: adminEmails.join(', '),
+            subject: util.format('%s has a new update available', app.fqdn),
+            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, format: 'text' })
+        };
+
+        enqueue(mailOptions);
+    });
+}
+
+function sendCrashNotification(program, context) {
+    assert.strictEqual(typeof program, 'string');
+    assert.strictEqual(typeof context, 'string');
+
+    var mailOptions = {
+        from: config.get('adminEmail'),
+        to: 'admin@cloudron.io',
+        subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
+        text: render('crash_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
+    };
+
+    enqueue(mailOptions);
+}
+
+function sendFeedback(user, type, subject, description) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof subject, 'string');
+    assert.strictEqual(typeof description, 'string');
+
+    assert(type === exports.FEEDBACK_TYPE_TICKET || type === exports.FEEDBACK_TYPE_FEEDBACK || type === exports.FEEDBACK_TYPE_APP);
+
+    var mailOptions = {
+        from: config.get('adminEmail'),
+        to: 'support@cloudron.io',
+        subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
+        text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})
+    };
+
+    enqueue(mailOptions);
+}

src/middleware/cors.js

@@ -0,0 +1,55 @@
+/* jshint node:true */
+
+'use strict';
+
+var url = require('url');
+
+/*
+ * CORS middleware
+ *
+ * options can contains a list of origins
+ */
+module.exports = function cors(options) {
+    options = options || { };
+    var maxAge = options.maxAge || 60 * 60 * 25 * 5; // 5 days
+    var origins = options.origins || [ '*' ];
+    var allowCredentials = options.allowCredentials || false; // cookies
+
+    return function (req, res, next) {
+        var requestOrigin = req.headers.origin;
+        if (!requestOrigin) return next();
+
+        requestOrigin = url.parse(requestOrigin);
+
+        var hostname = requestOrigin.host.split(':')[0]; // remove any port
+        var originAllowed = origins.some(function (o) { return o === '*' || o === hostname; });
+        if (!originAllowed) {
+            return res.status(405).send('CORS not allowed from this domain');
+        }
+
+        // respond back with req.headers.origin which might contain the scheme
+        res.header('Access-Control-Allow-Origin', req.headers.origin);
+        res.header('Access-Control-Allow-Credentials', allowCredentials);
+
+        // handle preflighted requests
+        if (req.method === 'OPTIONS') {
+            if (req.headers['access-control-request-method']) {
+                res.header('Access-Control-Allow-Methods', 'GET, PUT, DELETE, POST, OPTIONS');
+            }
+
+            if (req.headers['access-control-request-headers']) {
+                res.header('Access-Control-Allow-Headers', req.headers['access-control-request-headers']);
+            }
+
+            res.header('Access-Control-Max-Age', maxAge);
+
+            return res.status(200).send();
+        }
+
+        if (req.headers['access-control-request-headers']) {
+            res.header('Access-Control-Allow-Headers', req.headers['access-control-request-headers']);
+        }
+
+        next();
+    };
+};

src/middleware/index.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports = module.exports = {
+    cookieParser: require('cookie-parser'),
+    cors: require('./cors'),
+    csrf: require('csurf'),
+    favicon: require('serve-favicon'),
+    json: require('body-parser').json,
+    morgan: require('morgan'),
+    proxy: require('proxy-middleware'),
+    lastMile: require('connect-lastmile'),
+    multipart: require('./multipart.js'),
+    session: require('express-session'),
+    timeout: require('connect-timeout'),
+    urlencoded: require('body-parser').urlencoded
+};

src/middleware/multipart.js

@@ -0,0 +1,47 @@
+/* jshint node:true */
+
+'use strict';
+
+var multiparty = require('multiparty'),
+    timeout = require('connect-timeout');
+
+function _mime(req) {
+  var str = req.headers['content-type'] || '';
+  return str.split(';')[0];
+}
+
+module.exports = function multipart(options) {
+    return function (req, res, next) {
+        if (_mime(req) !== 'multipart/form-data') return res.status(400).send('Invalid content-type. Expecting multipart');
+
+        var form = new multiparty.Form({
+            uploadDir: '/tmp',
+            keepExtensions: true,
+            maxFieldsSize: options.maxFieldsSize || (2 * 1024), // only field size, not files
+            limit: options.limit || '8mb', // file sizes
+            autoFiles: true
+        });
+
+        // increase timeout of file uploads by default to 3 mins
+        if (req.clearTimeout) req.clearTimeout(); // clear any previous installed timeout middleware
+
+        timeout(options.timeout || (3 * 60 * 1000))(req, res, function () {
+            req.fields = { };
+            req.files = { };
+
+            form.parse(req, function (err, fields, files) {
+                if (err) return res.status(400).send('Error parsing request');
+                next(null);
+            });
+
+            form.on('file', function (name, file) {
+                req.files[name] = file;
+            });
+
+            form.on('field', function (name, value) {
+                req.fields[name] = value; // otherwise fields.name is an array
+            });
+        });
+    };
+};
+

src/oauth2views/callback.ejs

@@ -0,0 +1,45 @@
+<!-- callback tester -->
+
+<script>
+
+'use strict';
+
+var code = null;
+var redirectURI = null;
+var accessToken = null;
+var tokenType = null;
+var state = null;
+
+var args = window.location.search.slice(1).split('&');
+args.forEach(function (arg) {
+    var tmp = arg.split('=');
+    if (tmp[0] === 'redirectURI') {
+        redirectURI = window.decodeURIComponent(tmp[1]);
+    } else if (tmp[0] === 'code') {
+        code = window.decodeURIComponent(tmp[1]);
+    } else if (tmp[0] === 'state') {
+        state = window.decodeURIComponent(tmp[1]);
+    }
+});
+
+args = window.location.hash.slice(1).split('&');
+args.forEach(function (arg) {
+    var tmp = arg.split('=');
+    if (tmp[0] === 'access_token') {
+        accessToken = window.decodeURIComponent(tmp[1]);
+    } else if (tmp[0] === 'token_type') {
+        tokenType = window.decodeURIComponent(tmp[1]);
+    } else if (tmp[0] === 'state') {
+        state = window.decodeURIComponent(tmp[1]);
+    }
+});
+
+if (code && redirectURI) {
+    window.location.href = redirectURI + '?code=' + code + (state ? '&state=' + state : '');
+} else if (redirectURI && accessToken) {
+    window.location.href = redirectURI + '?token=' + accessToken + (state ? '&state=' + state : '');
+} else {
+    window.location.href = '/api/v1/session/login';
+}
+
+</script>

src/oauth2views/error.ejs

@@ -0,0 +1,26 @@
+<% include header %>
+
+<!-- error tester -->
+
+<br/>
+
+<div class="container">
+    <div class="row">
+        <div class="col-md-2"></div>
+        <div class="col-md-8">
+            <div class="alert alert-danger">
+                <%- message %>
+            </div>
+        </div>
+        <div class="col-md-2"></div>
+    </div>
+    <div class="row">
+        <div class="col-md-2"></div>
+        <div class="col-md-8 text-center">
+            <a href="<%- adminOrigin %>">Back</a>
+        </div>
+        <div class="col-md-2"></div>
+    </div>
+</div>
+
+<% include footer %>

src/oauth2views/footer.ejs

@@ -0,0 +1,3 @@
+
+    </body>
+</html>

src/oauth2views/header.ejs

@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
+
+    <title> Cloudron Login </title>
+
+    <!-- Custom Fonts -->
+    <link href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">
+    <link href="https://fonts.googleapis.com/css?family=Roboto:300" rel="stylesheet" type="text/css">
+
+    <!-- jQuery-->
+    <script src="<%= adminOrigin %>/3rdparty/js/jquery.min.js"></script>
+
+    <!-- Bootstrap Core JavaScript -->
+    <script src="<%= adminOrigin %>/3rdparty/js/bootstrap.min.js"></script>
+
+    <!-- Angularjs scripts -->
+    <script src="<%= adminOrigin %>/3rdparty/js/angular.min.js"></script>
+    <script src="<%= adminOrigin %>/3rdparty/js/angular-loader.min.js"></script>
+
+    <!-- Theme CSS -->
+    <link href="<%= adminOrigin %>/theme.css" rel="stylesheet">
+
+</head>
+
+<body class="oauth">
+
+    <!-- Navigation -->
+    <nav class="navbar navbar-default navbar-static-top shadow" role="navigation" style="margin-bottom: 0">
+        <div class="container-fluid">
+            <div class="navbar-header">
+                <span class="navbar-brand navbar-brand-icon"><img src="/api/v1/cloudron/avatar" width="40" height="40"/></span>
+                <span class="navbar-brand">Cloudron</span>
+            </div>
+        </div>
+    </nav>

src/oauth2views/login.ejs

@@ -0,0 +1,58 @@
+<% include header %>
+
+<!-- login tester -->
+
+<div class="container">
+    <div class="row">
+        <div class="col-md-6 col-md-offset-3">
+            <div class="card">
+                <div class="row">
+                    <div class="col-md-12" style="text-align: center;">
+                        <img width="128" height="128" src="<%= applicationLogo %>"/>
+                        <h1><small>Login to</small> <%= applicationName %></h1>
+                        <br/>
+                    </div>
+                </div>
+                <br/>
+            <% if (error) { %>
+                <div class="row">
+                    <div class="col-md-12">
+                        <h4 class="has-error"><%= error %></h4>
+                    </div>
+                </div>
+            <% } %>
+                <div class="row">
+                    <div class="col-md-12">
+                        <form id="loginForm" action="" method="post">
+                            <input type="hidden" name="_csrf" value="<%= csrf %>"/>
+                            <div class="form-group">
+                                <label class="control-label" for="inputUsername">Username or Email</label>
+                                <input type="text" class="form-control" id="inputUsername" name="username" autofocus required>
+                            </div>
+                            <div class="form-group">
+                                <label class="control-label" for="inputPassword">Password</label>
+                                <input type="password" class="form-control" name="password" id="inputPassword" required>
+                            </div>
+                            <input class="btn btn-primary btn-outline pull-right" type="submit" value="Sign in"/>
+                        </form>
+                        <a href="/api/v1/session/password/resetRequest.html">Reset your password</a>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script>
+
+(function () {
+    'use strict';
+
+    var search = window.location.search.slice(1).split('&').map(function (item) { return item.split('='); }).reduce(function (o, k) { o[k[0]] = k[1]; return o; }, {});
+
+    document.getElementById('loginForm').action = '/api/v1/session/login?returnTo=' + search.returnTo;
+})();
+
+</script>
+
+<% include footer %>