Add script to create a release in staging

Fixes #145

.gitattributes

@@ -1,6 +1,10 @@
 # Skip files when using git archive
 .gitattributes export-ignore
 .gitignore export-ignore
+/release export-ignore
 /scripts export-ignore
 test export-ignore
+/webadmin/src export-ignore
+/webadmin/deploymentConfig.json export-ignore
+/gulpfile.json export-ignore
 

.gitignore

@@ -1,9 +1,20 @@
 node_modules/
 coverage/
 docs/
-webadmin/dist/
-setup/splash/website/
 
-# vim swap files
+# vim swam files
 *.swp
 
+# supervisor
+supervisord.pid
+supervisord.log
+
+# nginx
+nginx/*.log
+nginx/*.pid
+nginx/naked_domain.conf
+nginx/applications/
+
+# release files
+release/versions-dev.json
+

README.md

@@ -4,7 +4,14 @@ The Box
 Development setup
 -----------------
 * sudo useradd -m yellowtent
+** This dummy user is required for supervisor 'box' configs
 ** Add admin-localhost as 127.0.0.1 in /etc/hosts
 ** All apps will be installed as hypened-subdomains of localhost. You should add
    hyphened-subdomains of your apps into /etc/hosts
 
+Running
+-------
+* `./run.sh` - this starts up nginx to serve up the webadmin
+* `DEBUG=box:* ./app.js` - this the main box code.
+* Navigate to https://admin-localhost
+

app.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+var server = require('./src/server.js'),
+    config = require('./config.js');
+
+console.log();
+console.log('==========================================');
+console.log(' Cloudron will use the following settings ');
+console.log('==========================================');
+console.log();
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : (config.LOCAL ? 'LOCAL' : 'TEST'));
+console.log(' Admin Origin:                   ', config.adminOrigin());
+console.log(' Appstore token:                 ', config.token());
+console.log(' Appstore server origin:         ', config.appServerUrl());
+console.log();
+console.log('==========================================');
+console.log();
+
+server.start(function (err) {
+    if (err) {
+        console.error('Error starting server', err);
+        process.exit(1);
+    }
+
+    console.log('Server listening on port ' + config.get('port'));
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () { server.stop(NOOP_CALLBACK); });
+process.on('SIGTERM', function () { server.stop(NOOP_CALLBACK); });

apphealthtask.js

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+var appdb = require('./src/appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    database = require('./src/database.js'),
+    DatabaseError = require('./src/databaseerror.js'),
+    debug = require('debug')('box:apphealthtask'),
+    docker = require('./src/docker.js'),
+    mailer = require('./src/mailer.js'),
+    os = require('os'),
+    superagent = require('superagent');
+
+exports = module.exports = {
+    initialize: initialize,
+    run: run
+};
+
+var FATAL_CALLBACK = function (error) {
+    if (!error) return;
+    console.error(error);
+    process.exit(2);
+};
+
+var HEALTHCHECK_INTERVAL = 30000;
+var gLastSeen = { }; // { time, emailSent }
+
+function initialize(callback) {
+    async.series([
+        database.initialize,
+        mailer.initialize
+    ], callback);
+}
+
+function setHealth(app, alive, runState, callback) {
+    assert(typeof app === 'object');
+    assert(typeof alive === 'boolean');
+    assert(typeof runState === 'string');
+    assert(typeof callback === 'function');
+
+    var healthy = true; // app is unhealthy if not alive for 2 mins
+    var now = new Date();
+
+    if (alive || !(app.id in gLastSeen)) { // give never seen apps 2 mins to come up
+        gLastSeen[app.id] = { time: now, emailSent: false };
+    } else if (Math.abs(now - gLastSeen[app.id].time) > 120 * 1000) { // not seen for 2 mins
+        debug('app %s not seen for more than 2 mins, marking as unhealthy', app.id);
+        healthy = false;
+    }
+
+    if (!healthy && !gLastSeen[app.id].emailSent) {
+        gLastSeen[app.id].emailSent = true;
+        mailer.appDied(app);
+    }
+
+    appdb.setHealth(app.id, healthy, runState, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled?
+        if (error) return callback(error);
+
+        app.healthy = healthy;
+        app.runState = runState;
+
+        callback(null);
+    });
+}
+
+
+// # TODO should probably poll from the outside network instead of the docker network?
+// callback is called with error for fatal errors and not if health check failed
+function checkAppHealth(app, callback) {
+    // only check status of installed apps. we could possibly optimize more by checking runState as well
+    if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(null);
+
+    var container = docker.getContainer(app.containerId),
+        manifest = app.manifest;
+
+    container.inspect(function (err, data) {
+        if (err || !data || !data.State) {
+            debug('Error inspecting container');
+            return setHealth(app, false, appdb.RSTATE_ERROR, callback);
+        }
+
+        if (data.State.Running !== true) {
+            debug('app %s has exited', app.id);
+            return setHealth(app, false, appdb.RSTATE_DEAD, callback);
+        }
+
+        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
+        superagent
+            .get(healthCheckUrl)
+            .timeout(HEALTHCHECK_INTERVAL)
+            .end(function (error, res) {
+
+            if (error || res.status !== 200) {
+                debug('app %s is not alive ', app.id);
+                setHealth(app, false, appdb.RSTATE_RUNNING, callback);
+            } else {
+                debug('app %s is alive', app.id);
+                setHealth(app, true, appdb.RSTATE_RUNNING, callback);
+            }
+        });
+    });
+}
+
+function processApps(callback) {
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(error);
+
+        async.each(apps, checkAppHealth, function (error) {
+            if (error) console.error(error);
+            callback(null);
+        });
+    });
+}
+
+function run(callback) {
+    processApps(function (error) {
+        if (error) return callback(error);
+        setTimeout(run.bind(null, callback), HEALTHCHECK_INTERVAL);
+    });
+}
+
+if (require.main === module) {
+    initialize();
+
+    run(function (error) {
+        console.error('apphealth task exiting with error.', error);
+        process.exit(error ? 1 : 0);
+    });
+}
+

box.js

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-// remove timestamp from debug() based output
-require('debug').formatArgs = function formatArgs() {
-    arguments[0] = this.namespace + ' ' + arguments[0];
-    return arguments;
-};
-
-var appHealthMonitor = require('./src/apphealthmonitor.js'),
-    async = require('async'),
-    config = require('./src/config.js'),
-    ldap = require('./src/ldap.js'),
-    oauthproxy = require('./src/oauthproxy.js'),
-    server = require('./src/server.js'),
-    simpleauth = require('./src/simpleauth.js');
-
-console.log();
-console.log('==========================================');
-console.log(' Cloudron will use the following settings ');
-console.log('==========================================');
-console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
-console.log(' Version:                        ', config.version());
-console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore API server origin:     ', config.apiServerOrigin());
-console.log(' Appstore Web server origin:     ', config.webServerOrigin());
-console.log();
-console.log('==========================================');
-console.log();
-
-async.series([
-    server.start,
-    ldap.start,
-    simpleauth.start,
-    appHealthMonitor.start,
-    oauthproxy.start
-], function (error) {
-    if (error) {
-        console.error('Error starting server', error);
-        process.exit(1);
-    }
-});
-
-var NOOP_CALLBACK = function () { };
-
-process.on('SIGINT', function () {
-    server.stop(NOOP_CALLBACK);
-    ldap.stop(NOOP_CALLBACK);
-    simpleauth.stop(NOOP_CALLBACK);
-    oauthproxy.stop(NOOP_CALLBACK);
-    setTimeout(process.exit.bind(process), 3000);
-});
-
-process.on('SIGTERM', function () {
-    server.stop(NOOP_CALLBACK);
-    ldap.stop(NOOP_CALLBACK);
-    simpleauth.stop(NOOP_CALLBACK);
-    oauthproxy.stop(NOOP_CALLBACK);
-    setTimeout(process.exit.bind(process), 3000);
-});

config.js

@@ -0,0 +1,143 @@
+/* jslint node: true */
+
+'use strict';
+
+var path = require('path'),
+    fs = require('fs'),
+    safe = require('safetydance'),
+    assert = require('assert'),
+    _ = require('underscore'),
+    path = require('path'),
+    mkdirp = require('mkdirp');
+
+exports = module.exports = {
+    baseDir: baseDir,
+    get: get,
+    set: set,
+
+    // ifdefs to check environment
+    CLOUDRON: process.env.NODE_ENV === 'cloudron',
+    TEST: process.env.NODE_ENV === 'test',
+    LOCAL: process.env.NODE_ENV === 'local' || !process.env.NODE_ENV,
+
+    // convenience getters
+    appServerUrl: appServerUrl,
+    fqdn: fqdn,
+    token: token,
+    version: version,
+    isCustomDomain: isCustomDomain,
+
+    // these values are derived
+    adminOrigin: adminOrigin,
+    appFqdn: appFqdn,
+    zoneName: zoneName
+};
+
+var homeDir = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
+
+var data = { };
+
+function baseDir() {
+    if (exports.CLOUDRON) return homeDir;
+    if (exports.TEST) return path.join(homeDir, '.yellowtenttest');
+    if (exports.LOCAL) return path.join(homeDir, '.yellowtent');
+}
+
+var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
+
+function saveSync() {
+    fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
+}
+
+(function initConfig() {
+    // setup defaults
+    if (exports.CLOUDRON) {
+        data.port = 3000;
+        data.appServerUrl = process.env.APP_SERVER_URL || null; // APP_SERVER_URL is set during bootstrap in the box's supervisor manifest
+    } else if (exports.TEST) {
+        data.port = 5454;
+        data.appServerUrl = 'http://localhost:6060'; // hock doesn't support https
+    } else if (exports.LOCAL) {
+        data.port = 3000;
+        data.appServerUrl = 'http://localhost:5050';
+    } else {
+        assert(false, 'Unknown environment. This should not happen!');
+    }
+
+    data.fqdn = 'localhost';
+
+    data.token = null;
+    data.mailServer = null;
+    data.mailUsername = null;
+    data.mailDnsRecordIds = [ ];
+    data.boxVersionsUrl = null;
+    data.version = null;
+    data.isCustomDomain = false;
+
+    if (safe.fs.existsSync(cloudronConfigFileName)) {
+        var existingData = safe.JSON.parse(safe.fs.readFileSync(cloudronConfigFileName, 'utf8'));
+        _.extend(data, existingData); // overwrite defaults with saved config
+        return;
+    }
+
+    mkdirp.sync(path.dirname(cloudronConfigFileName));
+    saveSync();
+})();
+
+// set(obj) or set(key, value)
+function set(key, value) {
+    if (typeof key === 'object') {
+        var obj = key;
+        for (var k in obj) {
+            assert(k in data, 'config.js is missing key "' + k + '"');
+            data[k] = obj[k];
+        }
+    } else {
+        assert(key in data, 'config.js is missing key "' + key + '"');
+        data[key] = value;
+    }
+    saveSync();
+}
+
+function get(key) {
+    assert(typeof key === 'string');
+
+    return safe.query(data, key);
+}
+
+function appServerUrl() {
+    return get('appServerUrl');
+}
+
+function fqdn() {
+    return get('fqdn');
+}
+
+function appFqdn(location) {
+    assert(typeof location === 'string');
+    return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
+}
+
+function adminOrigin() {
+    return 'https://' + appFqdn('admin');
+}
+
+function token() {
+    return get('token');
+}
+
+function version() {
+    return get('version');
+}
+
+function isCustomDomain() {
+    return get('isCustomDomain');
+}
+
+function zoneName() {
+    if (isCustomDomain()) return fqdn(); // the appstore sets up the custom domain as a zone
+
+    // for shared domain name, strip out the hostname
+    return fqdn().substr(fqdn().indexOf('.') + 1);
+}
+

crashnotifier.js

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();

gulpfile.js

@@ -2,158 +2,73 @@
 
 'use strict';
 
-var ejs = require('gulp-ejs'),
+var _ejs = require('ejs'),
+    ejs = require('gulp-ejs'),
     gulp = require('gulp'),
     del = require('del'),
+    path = require('path'),
     concat = require('gulp-concat'),
-    uglify = require('gulp-uglify'),
-    serve = require('gulp-serve'),
-    sass = require('gulp-sass'),
     sourcemaps = require('gulp-sourcemaps'),
-    minifyCSS = require('gulp-minify-css'),
-    autoprefixer = require('gulp-autoprefixer'),
-    argv = require('yargs').argv;
+    fs = require('fs');
 
-gulp.task('3rdparty', function () {
-    gulp.src([
-        'webadmin/src/3rdparty/**/*.js',
-        'webadmin/src/3rdparty/**/*.map',
-        'webadmin/src/3rdparty/**/*.css',
-        'webadmin/src/3rdparty/**/*.otf',
-        'webadmin/src/3rdparty/**/*.eot',
-        'webadmin/src/3rdparty/**/*.svg',
-        'webadmin/src/3rdparty/**/*.ttf',
-        'webadmin/src/3rdparty/**/*.woff',
-        'webadmin/src/3rdparty/**/*.woff2'
-        ])
-        .pipe(gulp.dest('webadmin/dist/3rdparty/'))
-        .pipe(gulp.dest('setup/splash/website/3rdparty'));
-
-    gulp.src('node_modules/bootstrap-sass/assets/javascripts/bootstrap.min.js')
-        .pipe(gulp.dest('webadmin/dist/3rdparty/js'))
-        .pipe(gulp.dest('setup/splash/website/3rdparty/js'));
-});
-
-
-// --------------
-// JavaScript
-// --------------
-
-gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
-
-var oauth = {
-    clientId: argv.clientId || 'cid-webadmin',
-    clientSecret: argv.clientSecret || 'unused',
-    apiOrigin: argv.apiOrigin || ''
+_ejs.filters.basename = function (obj) {
+    return path.basename(obj);
 };
 
-console.log();
-console.log('Using OAuth credentials:');
-console.log(' ClientId:     %s', oauth.clientId);
-console.log(' ClientSecret: %s', oauth.clientSecret);
-console.log(' Cloudron API: %s', oauth.apiOrigin || 'default');
-console.log();
+gulp.task('3rdparty', function () {
+    return gulp.src([
+            'webadmin/src/3rdparty/**/*.js',
+            'webadmin/src/3rdparty/**/*.css',
+            'webadmin/src/3rdparty/**/*.otf',
+            'webadmin/src/3rdparty/**/*.eot',
+            'webadmin/src/3rdparty/**/*.svg',
+            'webadmin/src/3rdparty/**/*.ttf',
+            'webadmin/src/3rdparty/**/*.woff',
+            'webadmin/src/3rdparty/**/*.js'
+        ])
+        .pipe(gulp.dest('webadmin/dist/3rdparty/'));
+});
 
 gulp.task('js-index', function () {
-    gulp.src([
-        'webadmin/src/js/index.js',
-        'webadmin/src/js/client.js',
-        'webadmin/src/js/appstore.js',
-        'webadmin/src/js/main.js',
-        'webadmin/src/views/*.js'
-        ])
-        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
+    return gulp.src(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'])
         .pipe(sourcemaps.init())
-        .pipe(concat('index.js', { newLine: ';' }))
-        .pipe(uglify())
+        .pipe(concat('index.js'))
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
 gulp.task('js-setup', function () {
-    gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
-        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
+    return gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
         .pipe(sourcemaps.init())
-        .pipe(concat('setup.js', { newLine: ';' }))
-        .pipe(uglify())
+        .pipe(concat('setup.js'))
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('js-error', function () {
-    gulp.src(['webadmin/src/js/error.js'])
-        .pipe(sourcemaps.init())
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'));
+gulp.task('js', ['js-index', 'js-setup'], function () {});
+
+gulp.task('htmlViews', function () {
+    return gulp.src('webadmin/src/views/*.html')
+        .pipe(gulp.dest('webadmin/dist/views'));
 });
 
-gulp.task('js-update', function () {
-    gulp.src(['webadmin/src/js/update.js'])
-        .pipe(sourcemaps.init())
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'))
-        .pipe(gulp.dest('setup/splash/website/js'));
+gulp.task('html_templates', function () {
+    var config = JSON.parse(fs.readFileSync('./webadmin/deploymentConfig.json'));
+
+    return gulp.src('webadmin/src/*.ejs')
+        .pipe(ejs(config, { ext: '.html' }))
+        .pipe(gulp.dest('webadmin/dist'));
 });
 
-
-// --------------
-// HTML
-// --------------
-
-gulp.task('html', ['html-views', 'html-update'], function () {
-    return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
+gulp.task('html', ['html_templates', 'htmlViews'], function () {
+    return gulp.src('webadmin/src/*.html')
+        .pipe(gulp.dest('webadmin/dist'));
 });
 
-gulp.task('html-update', function () {
-    return gulp.src(['webadmin/src/update.html']).pipe(gulp.dest('setup/splash/website'));
+gulp.task('clean', function (callback) {
+    del(['webadmin/dist'], callback);
 });
 
-gulp.task('html-views', function () {
-    return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
+gulp.task('default', ['clean'], function () {
+    gulp.start('html', 'js', '3rdparty');
 });
-
-// --------------
-// CSS
-// --------------
-
-gulp.task('css', function () {
-    return gulp.src('webadmin/src/*.scss')
-        .pipe(sourcemaps.init())
-        .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
-        .pipe(autoprefixer())
-        .pipe(minifyCSS())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist'))
-        .pipe(gulp.dest('setup/splash/website'));
-});
-
-gulp.task('images', function () {
-    return gulp.src('webadmin/src/img/**')
-        .pipe(gulp.dest('webadmin/dist/img'));
-});
-
-// --------------
-// Utilities
-// --------------
-
-gulp.task('watch', ['default'], function () {
-    gulp.watch(['webadmin/src/*.scss'], ['css']);
-    gulp.watch(['webadmin/src/img/*'], ['images']);
-    gulp.watch(['webadmin/src/**/*.html'], ['html']);
-    gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
-    gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
-    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
-    gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
-    gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
-    gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);
-});
-
-gulp.task('clean', function () {
-    del.sync(['webadmin/dist', 'setup/splash/website']);
-});
-
-gulp.task('default', ['clean', 'html', 'js', '3rdparty', 'images', 'css'], function () {});
-
-gulp.task('develop', ['watch'], serve({ root: 'webadmin/dist', port: 4000 }));

migrations/20141021192552-db-create.js

@@ -1,12 +1,8 @@
 var dbm = require('db-migrate');
 var type = dbm.dataType;
-var url = require('url');
 
 exports.up = function(db, callback) {
-    var dbName = url.parse(process.env.DATABASE_URL).path.substr(1); // remove slash
-
-    // by default, mysql collates case insensitively. 'utf8_general_cs' is not available
-    db.runSql('ALTER DATABASE ' + dbName + '  DEFAULT CHARACTER SET=utf8 DEFAULT COLLATE utf8_bin', callback);
+    callback();
 };
 
 exports.down = function(db, callback) {

migrations/20141021235251-clients-add-webadmin.js

@@ -0,0 +1,21 @@
+var dbm = require('db-migrate');
+var type = dbm.dataType;
+
+var uuid = require('node-uuid');
+
+exports.up = function(db, callback) {
+    var scopes = 'root,profile,users,apps,settings,roleAdmin';
+    var adminOrigin = 'https://admin-localhost';
+
+    // postinstall.sh creates the webadmin entry in production mode
+    if (process.env.NODE_ENV !== 'test') return callback(null);
+
+    db.runSql('INSERT INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) ' +
+              'VALUES (?, ?, ?, ?, ?, ?, ?)', [ uuid.v4(), 'webadmin', 'cid-webadmin', 'unused', 'WebAdmin', adminOrigin, scopes ],
+              callback);
+};
+
+exports.down = function(db, callback) {
+    // not sure what is meaningful here
+    callback(null);
+};

migrations/20141022153739-settings-add-naked-domain.js

@@ -0,0 +1,10 @@
+var dbm = require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('INSERT INTO settings (key, value) VALUES (?, ?)', [ 'naked_domain', null ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE FROM settings WHERE key=?', [ 'naked_domain' ], callback);
+};

migrations/20141128175100-add-addonconfigs.js

@@ -0,0 +1,15 @@
+var dbm = require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('CREATE TABLE appAddonConfigs(' +
+              ' appId VARCHAR(512) NOT NULL,' +
+              ' addonId VARCHAR(32) NOT NULL,' +
+              ' value VARCHAR(512) NOT NULL,' +
+              ' FOREIGN KEY(appId) REFERENCES apps(id))', callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE appAddonConfigs', callback);
+};
+

migrations/20150303114527-users-add-resetToken.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN resetToken VARCHAR(128) DEFAULT ""', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN resetToken', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150303160528-tokens-expires-bigint.js

@@ -1,20 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('DELETE FROM tokens', [], function (error) {
-        if (error) console.error(error);
-
-        db.runSql('ALTER TABLE tokens MODIFY expires BIGINT', [], function (error) {
-            if (error) console.error(error);
-            callback(error);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE tokens MODIFY expires VARCHAR(512)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20150303171203-authcodes-add-expiresAt.js

@@ -1,16 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE authcodes ADD COLUMN expiresAt BIGINT NOT NULL', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE authcodes DROP COLUMN expiresAt', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20150308054950-appportbindings-add-environmentVariable.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE appPortBindings ADD COLUMN environmentVariable VARCHAR(128) NOT NULL', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE appPortBindings DROP COLUMN environmentVariable', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150309023251-appportbindings-drop-containerport.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE appPortBindings DROP COLUMN containerPort', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE appPortBindings ADD COLUMN containerPort VARCHAR(5) NOT NULL', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150311120713-tokens-rename-userid-to-identifier.js

@@ -1,20 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('DELETE FROM tokens', [], function (error) {
-        if (error) console.error(error);
-
-        db.runSql('ALTER TABLE tokens CHANGE userId identifier VARCHAR(128) NOT NULL', [], function (error) {
-            if (error) console.error(error);
-            callback(error);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE tokens CHANGE identifier userId VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20150323044254-apps-drop-version.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN version', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN version VARCHAR(32)', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150323174906-apps-alter-health.js

@@ -1,16 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN healthy, ADD COLUMN health VARCHAR(128)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN health, ADD COLUMN healthy INTEGER', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20150326072904-apps-add-lastBackupId.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupId VARCHAR(128)', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupId', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150404093025-apps-add-createdAt.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN createdAt', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150430165335-settings-default-autoupdatepattern.js

@@ -1,12 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    // everyday at 1am
-    db.runSql('INSERT settings (name, value) VALUES("autoupdate_pattern", ?)', [ '00 00 1 * * *' ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DELETE * FROM settings WHERE name="autoupdate_pattern"', [ ], callback);
-}
-

migrations/20150430210225-settings-default-timezone.js

@@ -1,15 +0,0 @@
-dbm = dbm || require('db-migrate');
-var safe = require('safetydance');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
-    tz = tz ? tz.trim() : 'America/Los_Angeles';
-
-    db.runSql('INSERT settings (name, value) VALUES("time_zone", ?)', [ tz ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DELETE * FROM settings WHERE name="time_zone"', [ ], callback);
-};
-

migrations/20150615075901-users-add-unique-constraints.js

@@ -1,24 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-var async = require('async');
-
-exports.up = function(db, callback) {
-
-    // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address
-
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
-        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_username UNIQUE (username)'),
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
-        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_email UNIQUE (email)'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_username'),
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(512)'),
-        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_email'),
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(512)'),
-    ], callback);
-};

migrations/20150615134751-users-adjust-username-and-email-constraints.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL'),
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254) NOT NULL'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
-        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
-    ], callback);
-};

migrations/20150618192028-apps-add-lastManifestJson.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN lastManifestJson VARCHAR(2048)', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN lastManifestJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150710044740-apps-rename-lastBackupConfigJson.js

@@ -1,17 +0,0 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps CHANGE lastManifestJson lastBackupConfigJson VARCHAR(2048)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps CHANGE lastBackupConfigJson lastManifestJson VARCHAR(2048)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150710170847-apps-add-oldConfigJson.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN oldConfigJson VARCHAR(2048)', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN oldConfigJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20150719014338-settings-remove-all.js

@@ -1,10 +0,0 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('DELETE FROM settings', [ ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20151013073519-apps-add-oauthProxy.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20151015232915-clients-add-type.js

@@ -1,17 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'DELETE FROM clients'),
-        db.runSql.bind(db, 'ALTER TABLE clients ADD COLUMN type VARCHAR(16) NOT NULL'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE clients DROP COLUMN type', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20151016131005-apps-rename-accessRestriction.js

@@ -1,17 +0,0 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps CHANGE accessRestriction accessRestrictionJson VARCHAR(2048)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps CHANGE accessRestrictionJson accessRestriction VARCHAR(2048)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20151113091257-apps-alter-manifestJson.js

@@ -1,16 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY manifestJson TEXT', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(2048)', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20151126111337-apps-alter-jsons.js

@@ -1,19 +0,0 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson TEXT'),
-        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson TEXT'),
-        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson TEXT')
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson VARCHAR(2048)'),
-        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson VARCHAR(2048)'),
-        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson VARCHAR(2048)')
-    ], callback);
-};

migrations/initial-schema.sql

@@ -2,66 +2,64 @@ CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
     username VARCHAR(512) NOT NULL,
     email VARCHAR(512) NOT NULL,
-    password VARCHAR(1024) NOT NULL,
-    salt VARCHAR(512) NOT NULL,
+    _password VARCHAR(512) NOT NULL,
+    publicPem VARCHAR(2048) NOT NULL,
+    _privatePemCipher VARCHAR(2048) NOT NULL,
+    _salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS tokens(
-    accessToken VARCHAR(128) NOT NULL UNIQUE,
-    userId VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128),
+    accessToken VARCHAR(512) NOT NULL UNIQUE,
+    userId VARCHAR(512) NOT NULL,
+    clientId VARCHAR(512),
     scope VARCHAR(512) NOT NULL,
     expires VARCHAR(512) NOT NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE,
-    appId VARCHAR(128) NOT NULL,
+    id VARCHAR(512) NOT NULL UNIQUE,
+    appId VARCHAR(512) NOT NULL,
+    clientId VARCHAR(512) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
+    name VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS apps(
-    id VARCHAR(128) NOT NULL UNIQUE,
-    appStoreId VARCHAR(128) NOT NULL,
+    id VARCHAR(512) NOT NULL UNIQUE,
+    appStoreId VARCHAR(512) NOT NULL,
     version VARCHAR(32),
     installationState VARCHAR(512) NOT NULL,
     installationProgress VARCHAR(512),
     runState VARCHAR(512),
     healthy INTEGER,
     containerId VARCHAR(128),
-    manifestJson VARCHAR(2048),
+    manifestJson VARCHAR,
     httpPort INTEGER,
-    location VARCHAR(128) NOT NULL UNIQUE,
+    location VARCHAR(512) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
     accessRestriction VARCHAR(512),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
-    hostPort INTEGER NOT NULL UNIQUE,
+    hostPort VARCHAR(5) NOT NULL UNIQUE,
     containerPort VARCHAR(5) NOT NULL,
-    appId VARCHAR(128) NOT NULL,
+    appId VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
 CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(128) NOT NULL UNIQUE,
-    userId VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128) NOT NULL,
+    authCode VARCHAR(512) NOT NULL UNIQUE,
+    userId VARCHAR(512) NOT NULL,
+    clientId VARCHAR(512) NOT NULL,
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
-    name VARCHAR(128) NOT NULL UNIQUE,
+    key VARCHAR(512) NOT NULL UNIQUE,
     value VARCHAR(512),
-    PRIMARY KEY(name));
-
-CREATE TABLE IF NOT EXISTS appAddonConfigs(
-    appId VARCHAR(128) NOT NULL,
-    addonId VARCHAR(32) NOT NULL,
-    value VARCHAR(512) NOT NULL,
-    FOREIGN KEY(appId) REFERENCES apps(id));
+    PRIMARY KEY(key));
 

migrations/schema.sql

@@ -1,87 +1,74 @@
 #### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 #### This file is not used by any code and is here to document the latest schema
 
-#### General ideas
-#### Default char set is utf8 and DEFAULT COLLATE is utf8_bin. Collate affects comparisons in WHERE and ORDER
-#### Strict mode is enabled
-#### VARCHAR - stored as part of table row (use for strings)
-#### TEXT - stored offline from table row (use for strings)
-#### BLOB - stored offline from table row (use for binary data)
-#### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
-
 CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(254) NOT NULL UNIQUE,
-    email VARCHAR(254) NOT NULL UNIQUE,
-    password VARCHAR(1024) NOT NULL,
-    salt VARCHAR(512) NOT NULL,
+    username VARCHAR(512) NOT NULL,
+    email VARCHAR(512) NOT NULL,
+    _password VARCHAR(512) NOT NULL,
+    publicPem VARCHAR(2048) NOT NULL,
+    _privatePemCipher VARCHAR(2048) NOT NULL,
+    _salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS tokens(
-    accessToken VARCHAR(128) NOT NULL UNIQUE,
-    identifier VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128),
+    accessToken VARCHAR(512) NOT NULL UNIQUE,
+    userId VARCHAR(512) NOT NULL,
+    clientId VARCHAR(512),
     scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires VARCHAR(512) NOT NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
-    appId VARCHAR(128) NOT NULL,
-    type VARCHAR(16) NOT NULL,
+    id VARCHAR(512) NOT NULL UNIQUE,
+    appId VARCHAR(512) NOT NULL,
+    clientId VARCHAR(512) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
+    name VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS apps(
-    id VARCHAR(128) NOT NULL UNIQUE,
-    appStoreId VARCHAR(128) NOT NULL,
+    id VARCHAR(512) NOT NULL UNIQUE,
+    appStoreId VARCHAR(512) NOT NULL,
+    version VARCHAR(32),
     installationState VARCHAR(512) NOT NULL,
     installationProgress VARCHAR(512),
     runState VARCHAR(512),
-    health VARCHAR(128),
+    healthy INTEGER,
     containerId VARCHAR(128),
-    manifestJson TEXT,
-    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
-    location VARCHAR(128) NOT NULL UNIQUE,
+    manifestJson VARCHAR,
+    httpPort INTEGER,
+    location VARCHAR(512) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
-    oauthProxy BOOLEAN DEFAULT 0,
-    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
-    lastBackupId VARCHAR(128),
-    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
-
-    oldConfigJson TEXT, // used to pass old config for apptask
-
+    accessRestriction VARCHAR(512),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
-    hostPort INTEGER NOT NULL UNIQUE,
-    environmentVariable VARCHAR(128) NOT NULL,
-    appId VARCHAR(128) NOT NULL,
+    hostPort VARCHAR(5) NOT NULL UNIQUE,
+    containerPort VARCHAR(5) NOT NULL,
+    appId VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(512) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512),
+    FOREIGN KEY(appId) REFERENCES apps(id));
+
 CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(128) NOT NULL UNIQUE,
-    userId VARCHAR(128) NOT NULL,
-    clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    authCode VARCHAR(512) NOT NULL UNIQUE,
+    userId VARCHAR(512) NOT NULL,
+    clientId VARCHAR(512) NOT NULL,
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
-    name VARCHAR(128) NOT NULL UNIQUE,
+    key VARCHAR(512) NOT NULL UNIQUE,
     value VARCHAR(512),
-    PRIMARY KEY(name));
-
-CREATE TABLE IF NOT EXISTS appAddonConfigs(
-    appId VARCHAR(128) NOT NULL,
-    addonId VARCHAR(32) NOT NULL,
-    value VARCHAR(512) NOT NULL,
-    FOREIGN KEY(appId) REFERENCES apps(id));
+    PRIMARY KEY(key));
 

oauthproxy.js

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+var express = require('express'),
+    url = require('url'),
+    async = require('async'),
+    assert = require('assert'),
+    debug = require('debug')('box:proxy'),
+    proxy = require('proxy-middleware'),
+    session = require('cookie-session'),
+    database = require('./src/database.js'),
+    appdb = require('./src/appdb.js'),
+    clientdb = require('./src/clientdb.js'),
+    config = require('./config.js'),
+    http = require('http');
+
+var gSessions = {};
+var gProxyMiddlewareCache = {};
+var gApp = express();
+var gHttpServer = http.createServer(gApp);
+
+var CALLBACK_URI = '/callback';
+var PORT = 4000;
+
+function startServer(callback) {
+    assert(typeof callback === 'function');
+
+    gHttpServer.on('error', console.error);
+
+    gApp.use(session({
+        keys: ['blue', 'cheese', 'is', 'something']
+    }));
+
+    gApp.use(function (req, res, next) {
+        if (req.session && gSessions[req.session.sessid]) return next();
+
+        if (req.path === CALLBACK_URI) {
+            // FIXME we need to exchange the authCode and verify it
+            req.session.sessid = req.query.authCode;
+
+            // this is a simple in memory auth store
+            gSessions[req.session.sessid] = 'ok';
+
+            debug('user verified.');
+
+            // now redirect to the actual initially requested URL
+            res.redirect(req.session.returnTo);
+        } else {
+            var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
+
+            if (!Number.isFinite(port)) {
+                console.error('Failed to parse nginx proxy header to get app port.');
+                return res.send(500, 'Routing error. No forwarded port.');
+            }
+
+            debug('begin verifying user for app on port %s.', port);
+
+            appdb.getByHttpPort(port, function (error, result) {
+                if (error) {
+                    console.error('Unknown app.', error);
+                    return res.send(500, 'Unknown app.');
+                }
+
+                clientdb.getByAppId('proxy-' + result.id, function (error, result) {
+                    if (error) {
+                        console.error('Unkonwn OAuth client.', error);
+                        return res.send(500, 'Unknown OAuth client.');
+                    }
+
+                    req.session.port = port;
+                    req.session.returnTo =  result.redirectURI + req.path;
+
+                    var callbackURL = result.redirectURI + CALLBACK_URI;
+                    var scope = 'profile,roleUser';
+                    var clientId = result.clientId;
+                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + clientId + '&redirect_uri=' + callbackURL + '&scope=' + scope;
+
+                    debug('begin OAuth flow for client %s.', result.name);
+
+                    // begin the OAuth flow
+                    res.redirect(oauthLogin);
+                });
+            });
+        }
+    });
+
+    gApp.use(function (req, res, next) {
+        var port = req.session.port;
+
+        debug('proxy request for port %s with path %s.', port, req.path);
+
+        var proxyMiddleware = gProxyMiddlewareCache[port];
+        if (!proxyMiddleware) {
+            console.log('Adding proxy middleware for port %d', port);
+
+            proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
+            gProxyMiddlewareCache[port] = proxyMiddleware;
+        }
+
+        proxyMiddleware(req, res, next);
+    });
+
+    gHttpServer.listen(PORT, callback);
+}
+
+async.series([
+    database.initialize,
+    startServer
+], function (error) {
+    if (error) {
+        console.error('Failed to start proxy server.', error);
+        process.exit(1);
+    }
+
+    console.log('Proxy server listening...');
+});

package.json

@@ -1,101 +1,96 @@
 {
-  "name": "Cloudron",
-  "description": "Main code for a cloudron",
+  "name": "yellowtent",
+  "description": "Yellow tent",
   "version": "0.0.1",
   "private": "true",
   "author": {
-    "name": "Cloudron authors"
+    "name": "Yellow tent authors",
+    "email": "girish@forwardbias.in"
   },
   "repository": {
     "type": "git"
   },
   "engines": [
-    "node >=4.0.0 <=4.1.1"
+    "node >= 0.10.0"
   ],
+  "bin": {
+    "yellowtent": "./server.js"
+  },
   "dependencies": {
-    "async": "^1.2.1",
-    "attempt": "^1.0.1",
-    "aws-sdk": "^2.1.46",
-    "body-parser": "^1.13.1",
-    "bytes": "^2.1.0",
-    "cloudron-manifestformat": "^2.0.0",
+    "async": "^0.6.2",
+    "body-parser": "~1.9.3",
+    "commander": "^2.2.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
-    "connect-timeout": "^1.5.0",
-    "cookie-parser": "^1.3.5",
+    "connect-lastmile": "0.0.8",
+    "connect-timeout": "~1.4.0",
+    "cookie-parser": "1.1.0",
     "cookie-session": "^1.1.0",
-    "cron": "^1.0.9",
-    "csurf": "^1.6.6",
-    "db-migrate": "^0.9.2",
-    "debug": "^2.2.0",
-    "dockerode": "^2.2.2",
-    "ejs": "^2.2.4",
-    "ejs-cli": "^1.0.1",
-    "express": "^4.12.4",
-    "express-session": "^1.11.3",
-    "hat": "0.0.3",
-    "json": "^9.0.3",
-    "ldapjs": "^0.7.1",
-    "memorystream": "^0.3.0",
-    "mime": "^1.3.4",
-    "morgan": "^1.6.0",
-    "multiparty": "^4.1.2",
-    "mysql": "^2.7.0",
-    "native-dns": "^0.7.0",
-    "node-uuid": "^1.4.3",
-    "nodemailer": "^1.3.0",
-    "nodemailer-smtp-transport": "^1.0.3",
+    "csurf": "^1.6.1",
+    "db-migrate": "~0.7.1",
+    "debug": "~0.8.1",
+    "dockerode": "~2.0.5",
+    "ejs": "^1.0.0",
+    "encfs": "^0.1.1",
+    "express": "~4.2.0",
+    "express-session": "~1.1.0",
+    "js-yaml": "~3.2.2",
+    "json": "~9.0.1",
+    "memorystream": "~0.2.0",
+    "mime": "^1.2.11",
+    "mkdirp": "~0.3.5",
+    "morgan": "~1.0.1",
+    "multiparty": "http://registry.npmjs.org/multiparty/-/multiparty-4.0.0.tgz",
+    "native-dns": "~0.6.1",
+    "node-uuid": "^1.4.1",
+    "nodejs-disks": "~0.2.1",
+    "nodemailer": "~1.3.0",
+    "nodemailer-smtp-transport": "~0.1.13",
     "oauth2orize": "^1.0.1",
-    "once": "^1.3.2",
-    "passport": "^0.2.2",
+    "once": "^1.3.0",
+    "passport": "~0.2.1",
     "passport-http": "^0.2.2",
     "passport-http-bearer": "^1.0.1",
     "passport-local": "^1.0.0",
-    "passport-oauth2-client-password": "^0.1.2",
-    "password-generator": "^2.0.2",
-    "proxy-middleware": "^0.13.0",
-    "safetydance": "^0.1.0",
-    "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
-    "split": "^1.0.0",
-    "superagent": "^1.5.0",
-    "supererror": "^0.7.1",
-    "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
-    "underscore": "^1.7.0",
-    "ursa": "^0.9.1",
-    "valid-url": "^1.0.9",
-    "validator": "^4.4.0",
-    "x509": "^0.2.2"
+    "passport-oauth2-client-password": "~0.1.2",
+    "password-generator": "~0.2.3",
+    "proxy-middleware": "~0.5.1",
+    "readdirp": "^1.0.1",
+    "rimraf": "^2.2.6",
+    "safetydance": "0.0.12",
+    "semver": "~4.2.0",
+    "serve-favicon": "~2.1.7",
+    "split": "^0.3.0",
+    "sqlite3": "^3.0.0",
+    "superagent": "~0.17.0",
+    "supererror": "~0.6.0",
+    "underscore": "~1.7.0",
+    "ursa": "^0.8.0",
+    "validator": "~3.22.1"
   },
   "devDependencies": {
     "apidoc": "*",
-    "bootstrap-sass": "^3.3.3",
+    "aws-sdk": "~2.0.23",
     "del": "^1.1.1",
     "expect.js": "*",
-    "gulp": "^3.8.11",
-    "gulp-autoprefixer": "^2.3.0",
+    "gulp": "^3.8.10",
     "gulp-concat": "^2.4.3",
     "gulp-ejs": "^1.0.0",
-    "gulp-minify-css": "^1.1.3",
-    "gulp-sass": "^2.0.1",
-    "gulp-serve": "^1.0.0",
-    "gulp-sourcemaps": "^1.5.2",
-    "gulp-uglify": "^1.1.0",
-    "hock": "~1.2.0",
+    "gulp-sourcemaps": "^1.3.0",
+    "hock": "~0.2.5",
+    "husky": "~0.6.2",
     "istanbul": "*",
-    "js2xmlparser": "^1.0.0",
     "mocha": "*",
-    "nock": "^3.4.0",
-    "node-sass": "^3.0.0-alpha.0",
-    "redis": "^2.4.2",
-    "request": "^2.65.0",
-    "sinon": "^1.12.2",
-    "yargs": "^3.15.0"
+    "nock": "~0.43.1",
+    "redis": "~0.12.1",
+    "s3-cli": "~0.11.1",
+    "semver": "~4.2.0",
+    "sinon": "~1.10.3"
   },
   "scripts": {
-    "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
-    "migrate_test": "BOX_ENV=test DATABASE_URL=mysql://root:@localhost/boxtest node_modules/.bin/db-migrate up",
-    "test": "npm run migrate_test && src/test/setupTest && BOX_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
+    "create_testdb": "rm -rf $HOME/.yellowtenttest/*; mkdir -p $HOME/.yellowtenttest/data; NODE_ENV=test DATABASE_URL=sqlite3:///$HOME/.yellowtenttest/data/cloudron.sqlite node_modules/.bin/db-migrate up",
+    "migrate": "mkdir -p $HOME/.yellowtent/data; DATABASE_URL=sqlite3:///$HOME/.yellowtent/data/cloudron.sqlite node_modules/.bin/db-migrate up",
+    "migrate_data": "DATABASE_URL=sqlite3:///home/yellowtent/data/cloudron.sqlite db-migrate up",
+    "test": "scripts/checkInstall && npm run-script create_testdb && NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
     "postmerge": "/bin/true",
     "precommit": "/bin/true",
     "prepush": "npm test",

release/publish.sh

@@ -0,0 +1,144 @@
+#!/bin/bash
+
+set -eu
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+readonly JSON="${SOURCE_DIR}/node_modules/.bin/json"
+[ "$(uname -s)" == "Darwin" ] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+readonly VERSIONS_URL_DEV="https://s3.amazonaws.com/cloudron-releases/versions-dev.json"
+readonly VERSIONS_S3_URL_DEV="s3://cloudron-releases/versions-dev.json"
+readonly VERSIONS_URL_STAGING="https://s3.amazonaws.com/cloudron-releases/versions-staging.json"
+readonly VERSIONS_S3_URL_STAGING="s3://cloudron-releases/versions-staging.json"
+
+if [[ ! -f "${SOURCE_DIR}/../installer/scripts/digitalOceanFunctions.sh" ]]; then
+    echo "Could not locate digitalOceanFunctions.sh"
+    exit 1
+fi
+
+source "${SOURCE_DIR}/../installer/scripts/digitalOceanFunctions.sh"
+
+new_versions_file=""
+source_tarball_url=""
+image_id=""
+cmd=""
+new_version=""
+changelog="If I told you, I'd have to kill you"
+upgrade="autodetect"
+versions_url="${VERSIONS_URL_DEV}"
+versions_s3_url="${VERSIONS_S3_URL_DEV}"
+
+args=$($GNU_GETOPT -o "" -l "dev,staging,code:,image:,rerelease,new,list,revert,changelog:,release:,upgrade" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --dev) shift;;
+    --staging) versions_url="${VERSIONS_URL_STAGING}"; versions_s3_url="${VERSIONS_S3_URL_STAGING}"; shift;;
+    --code) source_tarball_url="$2"; shift 2;;
+    --image) image_id="$2"; shift 2;;
+    --rerelease) cmd="rerelease"; shift;;
+    --new) cmd="new"; shift;;
+    --release) cmd="release"; new_versions_file="$2"; shift 2;;
+    --list) cmd="list"; shift;;
+    --revert) cmd="revert"; shift;;
+    --changelog) changelog="$2"; shift 2;;
+    --upgrade) upgrade="true"; shift;;
+    --) shift; break;;
+    *) echo "Unknown option $2"; exit;;
+    esac
+done
+
+shift $(expr $OPTIND - 1)
+
+download_current() {
+    versions_url="$1"
+    # download the existing version file if the user hasn't provided one
+    local current_versions_file=$(mktemp -t box-versions 2>/dev/null || mktemp)
+
+    if ! wget -q -O "${current_versions_file}" "${versions_url}"; then
+        echo "Error downloading versions file"
+        exit 1
+    fi
+
+    echo "${current_versions_file}"
+}
+
+if [[ "${cmd}" == "list" ]]; then
+    cat "$(download_current "${versions_url}")"
+    exit 0
+elif [[ "${cmd}" == "release" ]]; then
+    if [[ ! -f "${new_versions_file}" ]]; then
+        echo "${new_versions_file} cannot be found"
+        exit 1
+    fi
+elif [[ "${cmd}" == "new" ]]; then
+    if [[ -z "${source_tarball_url}" || -z "${image_id}" ]]; then
+        echo "--code and --image is required"
+        exit 1
+    fi
+
+    new_version="0.0.1"
+    image_name=$(get_image_name "${image_id}")
+
+    new_versions_file=$(mktemp -t box-versions 2>/dev/null || mktemp)
+    cat > "${new_versions_file}" <<EOF
+    {
+        "0.0.1": {
+            "sourceTarballUrl": "${source_tarball_url}",
+            "imageId": ${image_id},
+            "imageName": "${image_name}",
+            "changelog": [ "Let's start at the very beginning, a very good way to start" ],
+            "date": "$(date -u)",
+            "next": null
+        }
+    }
+EOF
+elif [[ "${cmd}" == "revert" ]]; then
+    new_versions_file=$(download_current "${versions_url}")
+    last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 1)
+    second_last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 2 | head -n 1)
+
+    echo "Removing $last_version and making $second_last_version the last release"
+    $JSON -q -I -f "${new_versions_file}" -e "delete this['${last_version}']"
+    $JSON -q -I -f "${new_versions_file}" -e "this['${second_last_version}'].next = null"
+else
+    new_versions_file=$(download_current "${versions_url}")
+    # modify existing versions.json
+    if [[ -z "${source_tarball_url}" && -z "${image_id}" && "${cmd}" != "rerelease" ]]; then
+        echo "--code or --image is required"
+        exit 1
+    fi
+
+    readonly last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 1)
+    if [[ -z "${source_tarball_url}" ]]; then
+        source_tarball_url=$($JSON -f "${new_versions_file}" -D, "${last_version},sourceTarballUrl")
+        echo "Using the previous code url : ${source_tarball_url}"
+    fi
+    if [[ -z "${image_id}" ]]; then
+        image_id=$($JSON -f "${new_versions_file}" -D, "${last_version},imageId")
+        echo "Using the previous image id : ${image_id}"
+    fi
+
+    if [[ "${upgrade}" == "autodetect" ]]; then
+        old_image_id=$($JSON -f "${new_versions_file}" -D, "${last_version},imageId")
+        upgrade=$([[ "${old_image_id}" != "${image_id}" ]] && echo "true" || echo "false")
+    fi
+
+    new_version=$($SOURCE_DIR/node_modules/.bin/semver -i "${last_version}")
+    echo "Releasing version ${new_version}"
+    image_name=$(get_image_name "${image_id}")
+
+    $JSON -q -I -f "${new_versions_file}" -e "this['${last_version}'].next = '${new_version}'"
+    $JSON -q -I -f "${new_versions_file}" -e "this['${new_version}'] = { 'sourceTarballUrl': '${source_tarball_url}', 'imageId': ${image_id}, 'imageName': '${image_name}', 'changelog': [ \"${changelog}\" ], 'upgrade': ${upgrade}, 'date': '$(date -u)', 'next': null }"
+fi
+
+echo "Verifying new versions file"
+$SOURCE_DIR/release/verify.js "${new_versions_file}"
+
+echo "Uploading new versions file"
+$SOURCE_DIR/node_modules/.bin/s3-cli put --acl-public --default-mime-type "application/json" "${new_versions_file}" "${versions_s3_url}"
+
+cat "${new_versions_file}"
+

release/stage.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -eu
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+readonly JSON="${SOURCE_DIR}/node_modules/.bin/json"
+readonly SEMVER="${SOURCE_DIR}/node_modules/.bin/semver"
+[ $(uname -s) == "Darwin" ] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+readonly VERSIONS_URL_DEV="https://s3.amazonaws.com/cloudron-releases/versions-dev.json"
+readonly VERSIONS_URL_STAGING="https://s3.amazonaws.com/cloudron-releases/versions-staging.json"
+readonly VERSIONS_S3_URL_STAGING="s3://cloudron-releases/versions-staging.json"
+
+verify_tag() {
+    tag="$1"
+    git rev-parse --verify "tags/$1" 2>/dev/null
+}
+
+download() {
+    # download the existing version file if the user hasn't provided one
+    local tmp_file=$(mktemp -t stage 2>/dev/null || mktemp)
+
+    if wget -q -O "${tmp_file}" "${1}"; then
+        echo "${tmp_file}"
+    fi
+}
+
+read_changelog() {
+    version="$1"
+    changelog_file="${SOURCE_DIR}/release/changelogs/changelog-$1"
+
+    if [[ -f "${changelog_file}" ]]; then
+        cat "${changelog_file}" | grep -v "^#"
+    fi
+}
+
+if [[ $# -lt 1 ]]; then
+    echo "Usage: stage.sh <dev-version>"
+    exit 1
+fi
+
+dev_version="$1"
+
+dev_versions_file=$(download "${VERSIONS_URL_DEV}")
+if [[ -z "${dev_versions_file}" ]]; then
+    echo "Error downloading dev versions file"
+    exit 1
+fi
+
+dev_version_info=$($JSON -f "${dev_versions_file}" -D, "${dev_version}")
+if [[ -z "${dev_version_info}" ]]; then
+    echo "No such version in dev ${dev_versions_file} ${dev_version}"
+    exit 1
+fi
+
+staging_versions_file=$(download "${VERSIONS_URL_STAGING}") ## TODO: this can fail
+if [[ -z "${staging_versions_file}" ]]; then
+    echo "Creating new staging release file"
+    staging_versions_file=$(mktemp -t stage 2>/dev/null || mktemp)
+    echo "{}" > ${staging_versions_file}
+    readonly staging_last_version="0.0.0"
+    staging_new_version="0.0.1"
+    upgrade="false"
+else
+    readonly staging_last_version=$(cat "${staging_versions_file}" | $JSON -ka | tail -n 1)
+    staging_new_version=$($SEMVER -i "${staging_last_version}")
+    $JSON -q -I -f "${staging_versions_file}" -e "this['${staging_last_version}'].next = '${staging_new_version}'"
+
+    last_image_id=$($JSON -f "${staging_versions_file}" -D, "${staging_last_version},imageId")
+    new_image_id=$($JSON -f "${dev_versions_file}" -D, "${dev_version},imageId")
+
+    upgrade=$([[ "${last_image_id}" != "${new_image_id}" ]] && echo "true" || echo "false")
+fi
+
+#TODO: check if the tag matches the sha1 in the sourceTarballUrl
+if ! verify_tag "v${staging_new_version}"; then
+    echo "No git tag named v${staging_new_version} found"
+    exit 1
+fi
+
+changelog=$(read_changelog "${staging_new_version}")
+if [[ -z "${changelog}" ]]; then
+    echo "Missing changelog file or empty change log"
+    exit 1
+fi
+
+echo "Releasing version ${staging_new_version}"
+$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'] = ${dev_version_info}"
+#$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].changelog = '[ "${changelog}" ]'"
+$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].upgrade = ${upgrade}"
+$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].date = '$(date -u)'"
+$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].next = null"
+
+echo "Verifying new versions file"
+$SOURCE_DIR/release/verify.js "${staging_versions_file}"
+
+echo "Uploading new versions file"
+$SOURCE_DIR/node_modules/.bin/s3-cli put --acl-public --default-mime-type "application/json" "${staging_versions_file}" "${VERSIONS_S3_URL_STAGING}"
+
+cat "${staging_versions_file}" | tee $SOURCE_DIR/release/versions-staging.json
+

release/verify.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+
+var AWS = require('aws-sdk'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    url = require('url');
+
+function die(msg) {
+    console.error(msg);
+    process.exit(1);
+}
+
+function verify(versionsFileName) {
+    // check if the json is valid
+    var versionsJson = safe.JSON.parse(fs.readFileSync(versionsFileName));
+    if (!versionsJson) {
+        die(versionsFileName + ' is not valid json : ' + safe.error);
+    }
+
+    // check all the keys
+    var sortedVersions = Object.keys(versionsJson).sort();
+    sortedVersions.forEach(function (version, index) {
+        if (typeof versionsJson[version].imageId !== 'number') die('version ' + version + ' does not have proper imageId');
+        if (typeof versionsJson[version].imageName !== 'string' || !versionsJson[version].imageName.length) die('version ' + version + ' does not have proper imageName');
+        if ('changeLog' in versionsJson[version] && !util.isArray(versionsJson[version].changeLog)) die('version ' + version + ' does not have proper changeLog');
+        if (typeof versionsJson[version].date !== 'string' || ((new Date(versionsJson[version].date)).toString() === 'Invalid Date')) die('invalid date or missing date');
+        if (versionsJson[version].next !== null && typeof versionsJson[version].next !== 'string') die('version ' + version + ' does not have proper next');
+        if (typeof versionsJson[version].sourceTarballUrl !== 'string') die('version ' + version + ' does not have proper sourceTarballUrl');
+        var tarballUrl = url.parse(versionsJson[version].sourceTarballUrl);
+        if (tarballUrl.protocol !== 'https:') die('sourceTarballUrl must be https');
+        if (!/.tar.gz$/.test(tarballUrl.path)) die('sourceTarballUrl must be tar.gz');
+
+        var nextVersion = versionsJson[version].next;
+        // despite having the 'next' field, the appstore code currently relies on all versions being sorted based on semver.compare (see boxversions.js)
+        if (nextVersion && semver.gt(version, nextVersion)) die('next version cannot be less than current @' + version);
+    });
+
+    // check that package.json version is in versions.json
+    var currentVersion = require('../package.json').version;
+    if (sortedVersions.indexOf(currentVersion) === -1) {
+        die('package.json version is not present in versions.json');
+    }
+}
+
+if (process.argv.length === 3) {
+    verify(process.argv[2]);
+    process.exit(0);
+} else {
+    console.log('verify.js <versions_file>');
+}
+
+
+

release/versions-stable.json

@@ -0,0 +1,7 @@
+{
+    "0.0.1": {
+        "revision": "9f09f8e7a8633e8b3341bb9c610f5f631ccd288c",
+        "imageId": 7531071,
+        "next": null
+    }
+}

run.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+echo
+echo "Starting Cloudron at port 443"
+echo
+
+readonly BOX_SRC_DIR="$(cd $(dirname "$0"); pwd)"
+readonly NGINX_ROOT=~/.yellowtent/nginx
+readonly PROVISION_VERSION=0.1
+readonly PROVISION_BOX_VERSIONS_URL=0.1
+readonly DATA_DIR=~/.yellowtent/data
+readonly FQDN=admin-localhost
+
+mkdir -p "${NGINX_ROOT}/applications"
+mkdir -p "${NGINX_ROOT}/cert"
+mkdir -p "${DATA_DIR}"
+
+# get the database current
+npm run-script migrate
+
+cp setup/start/nginx/nginx.conf "${NGINX_ROOT}/nginx.conf"
+cp setup/start/nginx/mime.types "${NGINX_ROOT}/mime.types"
+cp setup/start/nginx/cert/* "${NGINX_ROOT}/cert/"
+
+# adjust the generated nginx config for local use
+touch "${NGINX_ROOT}/naked_domain.conf"
+sed -e "s/##ADMIN_FQDN##/${FQDN}/" -e "s|##BOX_SRC_DIR##|${BOX_SRC_DIR}|" setup/start/nginx/admin.conf_template > "${NGINX_ROOT}/applications/admin.conf"
+sed -e "s/user www-data/user ${USER}/" -i "${NGINX_ROOT}/nginx.conf"
+
+# add webadmin oauth client
+readonly WEBADMIN_ID=abcdefg
+readonly WEBADMIN_SCOPES="root,profile,users,apps,settings,roleAdmin"
+sqlite3 "${DATA_DIR}/cloudron.sqlite" "INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES (\"${WEBADMIN_ID}\", \"webadmin\", \"cid-webadmin\", \"secret-webadmin\", \"WebAdmin\", \"https://${FQDN}\", \"${WEBADMIN_SCOPES}\")"
+
+# start nginx
+sudo nginx -c nginx.conf -p "${NGINX_ROOT}"
+

scripts/checkInstall

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eu
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+# reset sudo timestamp to avoid wrong success
+sudo -k || sudo --reset-timestamp
+
+# checks if all scripts are sudo access
+scripts=("${SOURCE_DIR}/src/scripts/rmappdir.sh" \
+         "${SOURCE_DIR}/src/scripts/reloadnginx.sh" \
+         "${SOURCE_DIR}/src/scripts/backup.sh" \
+         "${SOURCE_DIR}/src/scripts/reboot.sh" \
+         "${SOURCE_DIR}/src/scripts/reloadcollectd.sh")
+
+for script in "${scripts[@]}"; do
+    if [[ $(sudo -n "${script}" --check 2>/dev/null) != "OK" ]]; then
+        echo ""
+        echo "${script} does not have sudo access."
+        echo "You have to add the lines below to /etc/sudoers.d/yellowtent."
+        echo ""
+        echo "Defaults!${script} env_keep=HOME"
+        echo "${USER} ALL=(ALL) NOPASSWD: ${script}"
+        echo ""
+        exit 1
+    fi
+done
+
+if ! docker inspect girish/test:0.6 >/dev/null 2>/dev/null; then
+    echo "docker pull girish/test:0.6 for tests to run"
+    exit 1
+fi
+
+if ! docker inspect girish/redis:0.1 >/dev/null 2>/dev/null; then
+    echo "docker pull girish/redis:0.1 for tests to run"
+    exit 1
+fi
+
+exit 0

scripts/createSourceTarball

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -eu
+
+[[ ! -f "${HOME}/.s3cfg" ]] && echo "~/.s3cfg missing" && exit 1
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revision:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+commitish="HEAD"
+
+while true; do
+    case "$1" in
+    --revision) commitish="$2"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint?
+
+version=$(cd "${SOURCE_DIR}" && git rev-parse "${commitish}")
+bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
+bundle_file="${TMPDIR}/box-${version}.tar.gz"
+
+chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
+echo "Checking out code [${version}] into ${bundle_dir}"
+(cd "${SOURCE_DIR}" && git archive --format=tar HEAD | (cd "${bundle_dir}" && tar xf -))
+
+echo "Installing modules"
+cd "${bundle_dir}" && npm install --production
+
+cd "${bundle_dir}" && tar czvf "${bundle_file}" .
+
+echo "Uploading bundle to S3"
+${SOURCE_DIR}/node_modules/.bin/s3-cli put --acl-public "${bundle_file}" "s3://cloudron-releases/box-${version}.tar.gz"
+
+echo "Cleaning up ${bundle_dir}"
+rm -rf "${bundle_dir}" "${bundle_file}"
+

setup/DESIGN.md

@@ -1,57 +0,0 @@
-This document gives the design of this setup code.
-
-box code should be delivered in the form of a (docker) container.
-This is not the case currently but we want to do structure the code
-in spirit that way.
-
-### container.sh
-This contains code that essential goes into Dockerfile.
-
-This file contains static configuration over a base image. Currently,
-the yellowtent user is created in the installer base image but it
-could very well be placed here.
-
-The idea is that the installer would simply remove the old box container
-and replace it with a new one for an update.
-
-Because we do not package things as Docker yet, we should be careful
-about the code here. We have to expect remains of an older setup code.
-For example, older systemd or nginx configs might be around.
-
-The config directory is _part_ of the container and is not a VOLUME.
-Which is to say that the files will be nuked from one update to the next.
-
-The data directory is a VOLUME. Contents of this directory are expected
-to survive an update. This is a good place to place config files that
-are "dynamic" and need to survive restarts. For example, the infra
-version (see below) or the mysql/postgresql data etc.
-
-### start.sh
-  * It is called in 3 modes - new, update, restore.
-
-  * The first thing this does is to do the static container.sh setup.
-
-  * It then downloads any box restore data and restores the box db from the
-    backup.
-
-  * It then proceeds to call the db-migrate script.
-
-  * It then does dynamic configuration like setting up nginx, collectd.
-
-  * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
-
-  * box services are then started
-
-setup_infra.sh
-This setups containers like graphite, mail and the addons containers.
-
-Containers are relaunched based on the INFRA_VERSION. The script compares
-the version here with the version in the file DATA_DIR/INFRA_VERSION.
-
-If they match, the containers are not recreated and nothing is to be done.
-nginx, collectd configs are part of data already and containers are running.
-
-If they do not match, it deletes all containers (including app containers) and starts
-them all afresh. Important thing here is that, DATA_DIR is never removed across
-updates. So, it is only the containers being recreated and not the data.
-

setup/INFRA_VERSION

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=21
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.8.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.8.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.9.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -3,77 +3,34 @@
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 json="${script_dir}/../node_modules/.bin/json"
 
-# IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
-arg_api_server_origin=""
 arg_box_versions_url=""
-arg_fqdn=""
-arg_is_custom_domain="false"
-arg_restore_key=""
-arg_restore_url=""
-arg_retire="false"
-arg_tls_config=""
 arg_tls_cert=""
 arg_tls_key=""
+arg_app_server_url=""
+arg_fqdn=""
 arg_token=""
 arg_version=""
-arg_web_server_origin=""
-arg_backup_config=""
-arg_dns_config=""
-arg_provider=""
+arg_is_custom_domain="false"
 
-args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
+args=$(getopt -o "" -l "boxversionsurl:,data:,version:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
-    --retire)
-        arg_retire="true"
-        shift
-        ;;
+    --boxversionsurl) arg_box_versions_url="$2";;
     --data)
-        # only read mandatory non-empty parameters here
-        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_is_custom_domain arg_box_versions_url arg_version <<EOF
-        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
+        read -r arg_app_server_url arg_fqdn arg_token arg_is_custom_domain <<EOF
+        $(echo "$2" | $json appServerUrl fqdn token isCustomDomain | tr '\n' ' ')
 EOF
-        # read possibly empty parameters here
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
-        arg_token=$(echo "$2" | $json token)
-        arg_provider=$(echo "$2" | $json provider)
-
-        arg_tls_config=$(echo "$2" | $json tlsConfig)
-        [[ "${arg_tls_config}" == "null" ]] && arg_tls_config=""
-
-        arg_restore_url=$(echo "$2" | $json restore.url)
-        [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
-
-        arg_restore_key=$(echo "$2" | $json restore.key)
-        [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
-
-        arg_backup_config=$(echo "$2" | $json backupConfig)
-        [[ "${arg_backup_config}" == "null" ]] && arg_backup_config=""
-
-        arg_dns_config=$(echo "$2" | $json dnsConfig)
-        [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
-
-        shift 2
         ;;
+    --version) arg_version="$2";;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
     esac
+
+    shift 2
 done
 
-echo "Parsed arguments:"
-echo "api server: ${arg_api_server_origin}"
-echo "box versions url: ${arg_box_versions_url}"
-echo "fqdn: ${arg_fqdn}"
-echo "custom domain: ${arg_is_custom_domain}"
-echo "restore key: ${arg_restore_key}"
-echo "restore url: ${arg_restore_url}"
-echo "tls cert: ${arg_tls_cert}"
-echo "tls key: ${arg_tls_key}"
-echo "token: ${arg_token}"
-echo "tlsConfig: ${arg_tls_config}"
-echo "version: ${arg_version}"
-echo "web server: ${arg_web_server_origin}"
-echo "provider: ${arg_provider}"
+

setup/container.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-# This file can be used in Dockerfile
-
-readonly container_files="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/container"
-
-readonly CONFIG_DIR="/home/yellowtent/configs"
-readonly DATA_DIR="/home/yellowtent/data"
-
-########## create config directory
-rm -rf "${CONFIG_DIR}"
-sudo -u yellowtent mkdir "${CONFIG_DIR}"
-
-########## systemd
-rm -f /etc/systemd/system/janitor.*
-cp -r "${container_files}/systemd/." /etc/systemd/system/
-systemctl daemon-reload
-systemctl enable cloudron.target
-
-########## sudoers
-rm -f /etc/sudoers.d/yellowtent
-cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
-
-########## collectd
-rm -rf /etc/collectd
-ln -sfF "${DATA_DIR}/collectd" /etc/collectd
-
-########## apparmor docker profile
-cp "${container_files}/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
-systemctl restart apparmor
-
-########## nginx
-# link nginx config to system config
-unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
-ln -s "${DATA_DIR}/nginx" /etc/nginx
-
-########## mysql
-cp "${container_files}/mysql.cnf" /etc/mysql/mysql.cnf
-
-########## Enable services
-update-rc.d -f collectd defaults
-

setup/container/docker-cloudron-app.apparmor

@@ -1,32 +0,0 @@
-#include <tunables/global>
-
-
-profile docker-cloudron-app flags=(attach_disconnected,mediate_deleted) {
-
-  #include <abstractions/base>
-
-  ptrace peer=@{profile_name},
-
-  network,
-  capability,
-  file,
-  umount,
-
-  deny @{PROC}/sys/fs/** wklx,
-  deny @{PROC}/sysrq-trigger rwklx,
-  deny @{PROC}/mem rwklx,
-  deny @{PROC}/kmem rwklx,
-  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
-  deny @{PROC}/sys/kernel/*/** wklx,
-
-  deny mount,
-
-  deny /sys/[^f]*/** wklx,
-  deny /sys/f[^s]*/** wklx,
-  deny /sys/fs/[^c]*/** wklx,
-  deny /sys/fs/c[^g]*/** wklx,
-  deny /sys/fs/cg[^r]*/** wklx,
-  deny /sys/firmware/efi/efivars/** rwklx,
-  deny /sys/kernel/security/** rwklx,
-}
-

setup/container/mysql.cnf

@@ -1,7 +0,0 @@
-!includedir /etc/mysql/conf.d/
-!includedir /etc/mysql/mysql.conf.d/
-
-# http://bugs.mysql.com/bug.php?id=68514
-[mysqld]
-performance_schema=OFF
-max_connection=50

setup/container/sudoers

@@ -1,32 +0,0 @@
-# sudo logging breaks journalctl output with very long urls (systemd bug)
-Defaults !syslog
-
-Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
-
-Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmappdir.sh
-
-Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
-
-Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupbox.sh
-
-Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupapp.sh
-
-Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restoreapp.sh
-
-Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
-
-Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
-
-Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
-
-Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

setup/container/systemd/box.service

@@ -1,19 +0,0 @@
-[Unit]
-Description=Cloudron Admin
-OnFailure=crashnotifier@%n.service
-StopWhenUnneeded=true
-
-[Service]
-Type=idle
-WorkingDirectory=/home/yellowtent/box
-Restart=always
-ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
-KillMode=process
-User=yellowtent
-Group=yellowtent
-MemoryLimit=200M
-TimeoutStopSec=5s
-StartLimitInterval=1
-StartLimitBurst=60
-

setup/container/systemd/cloudron.target

@@ -1,10 +0,0 @@
-[Unit]
-Description=Cloudron Smart Cloud
-Documentation=https://cloudron.io/documentation.html
-StopWhenUnneeded=true
-Requires=box.service
-After=box.service
-# AllowIsolate=yes
-
-[Install]
-WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -1,15 +0,0 @@
-# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
-[Unit]
-Description=Cloudron Crash Notifier for %i
-# otherwise, systemd will kill this unit immediately as nobody requires it
-StopWhenUnneeded=false
-
-[Service]
-Type=idle
-WorkingDirectory=/home/yellowtent/box
-ExecStart="/home/yellowtent/box/crashnotifier.js" %I
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
-KillMode=process
-User=yellowtent
-Group=yellowtent
-MemoryLimit=50M

setup/splash/nginx/mime.types

@@ -0,0 +1,80 @@
+
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/x-javascript              js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg svgz;
+    image/webp                            webp;
+
+    application/java-archive              jar war ear;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/xhtml+xml                 xhtml;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              eot;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/ogg                             ogg;
+    audio/x-m4a                           m4a;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mp4                             mp4;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/webm                            webm;
+    video/x-flv                           flv;
+    video/x-m4v                           m4v;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

setup/splash/nginx/nginx.conf

@@ -0,0 +1,63 @@
+user www-data;
+worker_processes 1;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include mime.types;
+    default_type application/octet-stream;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    root ##SETUP_WEBSITE_DIR##;
+    index index.html;
+
+    server {
+        listen 80;
+
+        location / {
+            # redirect everything to HTTPS
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    # HTTPS server
+    server {
+        listen 443;
+
+        error_page 503 /index.html;
+
+        location /index.html {
+            # allow access to this page
+            add_header Cache-Control no-cache;
+        }
+
+        location /3rdparty/bootstrap.min.css {
+            # allow access to this page
+            add_header Cache-Control no-cache;
+        }
+
+        location /progress.json {
+            # allow access to this page
+            add_header Cache-Control no-cache;
+        }
+
+        location / {
+            return 503;
+        }
+
+        ssl on;
+        ssl_certificate      cert/host.cert;
+        ssl_certificate_key  cert/host.key;
+
+        ssl_session_timeout 5m;
+
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+        ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
+        ssl_prefer_server_ciphers on;
+    }
+}

setup/splash/website/index.html

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
+
+    <title> Cloudron Webadmin </title>
+
+    <link href="3rdparty/bootstrap.min.css" rel="stylesheet">
+</head>
+
+<body>
+    <!-- Modal update progress -->
+    <div class="modal show" id="updateProgressModal" tabindex="-1" role="dialog" aria-labelledby="updateProgressModalLabel" aria-hidden="true" data-keyboard    ="false" data-backdrop="static">
+        <div class="modal-dialog">
+            <div class="modal-content">
+                <div class="modal-header">
+                    <h4 class="modal-title" id="updateProgressModalLabel">Update in progress...</h4>
+                </div>
+                <div class="modal-body">
+                    <div class="progress progress-striped active">
+                        <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%"></div>
+                    </div>
+                </div>
+            </div>
+            <!-- /.modal-content -->
+        </div>
+        <!-- /.modal-dialog -->
+    </div>
+
+    <script>
+        setTimeout(location.reload.bind(location, true /* forceGet from server */), 10000);
+    </script>
+</body>
+
+</html>

setup/splashpage.sh

@@ -1,40 +1,32 @@
 #!/bin/bash
 
-set -eu -o pipefail
+set -eu
 
+readonly NGINX_CONFIG_DIR="/home/yellowtent/setup/configs/nginx" # do not reuse configs since it will be removed by installer
 readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly BOX_SRC_DIR="/home/yellowtent/box"
-readonly DATA_DIR="/home/yellowtent/data"
-readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
-
-source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
 
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
-# keep this is sync with config.js appFqdn()
-admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
-admin_origin="https://${admin_fqdn}"
-
 # copy the website
 rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
-    rm -f ${DATA_DIR}/nginx/applications/*
-    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
-else
-    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
-fi
+rm -rf "${NGINX_CONFIG_DIR}" && mkdir -p "${NGINX_CONFIG_DIR}"
+sed -e "s|##SETUP_WEBSITE_DIR##|${SETUP_WEBSITE_DIR}|" "${script_dir}/splash/nginx/nginx.conf" > "${NGINX_CONFIG_DIR}/nginx.conf"
+cp "${script_dir}/splash/nginx/mime.types" "${NGINX_CONFIG_DIR}/mime.types"
+mkdir -p "${NGINX_CONFIG_DIR}/cert"
+echo "${arg_tls_cert}" > "${NGINX_CONFIG_DIR}/cert/host.cert"
+echo "${arg_tls_key}" > "${NGINX_CONFIG_DIR}/cert/host.key"
 
-echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+# link in the new nginx config
+unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
+ln -s "${NGINX_CONFIG_DIR}" /etc/nginx
+
+touch "${SETUP_WEBSITE_DIR}/progress.json"
 
 nginx -s reload

setup/start.sh

@@ -1,213 +1,196 @@
 #!/bin/bash
 
-set -eu -o pipefail
+# Count installer files so that we can correlate install and postinstall logs
+install_count=$(find /var/log/cloudron -name "installer*" | wc -l)
+exec > >(tee "/var/log/cloudron/start-$install_count.log")
+exec 2>&1
+
+set -eux
 
 echo "==== Cloudron Start ===="
 
 readonly USER="yellowtent"
+# NOTE: Do NOT use BOX_SRC_DIR for accessing code and config files. This script will be run from a temp directory
+# and the whole code will relocated to BOX_SRC_DIR by the installer. Use paths relative to script_dir or box_src_tmp_dir
 readonly BOX_SRC_DIR="/home/${USER}/box"
 readonly DATA_DIR="/home/${USER}/data"
 readonly CONFIG_DIR="/home/${USER}/configs"
+readonly MAIL_SERVER_IP="172.17.120.120" # hardcoded in haraka container
 readonly SETUP_PROGRESS_JSON="/home/yellowtent/setup/website/progress.json"
-readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
-
-readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+box_src_tmp_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
-# keep this is sync with config.js appFqdn()
-admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
-admin_origin="https://${admin_fqdn}"
-
-readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "admin.${arg_fqdn}" ||  echo "admin-${arg_fqdn}")
 
 set_progress() {
-    local percent="$1"
+    local progress="$1"
     local message="$2"
 
-    echo "==== ${percent} - ${message} ===="
-    (echo "{ \"update\": { \"percent\": \"${percent}\", \"message\": \"${message}\" }, \"backup\": {} }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
+    echo "==== ${message} ===="
+    (echo "{ \"progress\": \"${progress}\", \"message\": \"${message}\" }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
 }
 
-set_progress "1" "Create container"
-$script_dir/container.sh
+set_progress "5" "Configuring Sudoers file"
+cat > /etc/sudoers.d/yellowtent <<EOF
+Defaults!${BOX_SRC_DIR}/src/scripts/rmappdir.sh env_keep=HOME
+${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/rmappdir.sh
 
-set_progress "10" "Ensuring directories"
-# keep these in sync with paths.js
-[[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
-mkdir -p "${DATA_DIR}/box/appicons"
-mkdir -p "${DATA_DIR}/box/certs"
-mkdir -p "${DATA_DIR}/box/mail"
-mkdir -p "${DATA_DIR}/box/acme" # acme keys
-mkdir -p "${DATA_DIR}/graphite"
+Defaults!${BOX_SRC_DIR}/src/scripts/reloadnginx.sh env_keep=HOME
+${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reloadnginx.sh
 
-mkdir -p "${DATA_DIR}/mysql"
-mkdir -p "${DATA_DIR}/postgresql"
-mkdir -p "${DATA_DIR}/mongodb"
-mkdir -p "${DATA_DIR}/snapshots"
-mkdir -p "${DATA_DIR}/addons"
-mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
-mkdir -p "${DATA_DIR}/acme" # acme challenges
+Defaults!${BOX_SRC_DIR}/src/scripts/backup.sh env_keep=HOME
+${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/backup.sh
 
-# bookkeep the version as part of data
-echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
+Defaults!${BOX_SRC_DIR}/src/scripts/reboot.sh env_keep=HOME
+${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reboot.sh
 
-# remove old snapshots. if we do want to keep this around, we will have to fix the chown -R below
-# which currently fails because these are readonly fs
-echo "Cleaning up snapshots"
-find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
+Defaults!${BOX_SRC_DIR}/src/scripts/reloadcollectd.sh env_keep=HOME
+${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reloadcollectd.sh
 
-# restart mysql to make sure it has latest config
-service mysql restart
-
-readonly mysql_root_password="password"
-mysqladmin -u root -ppassword password password # reset default root password
-mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
-
-if [[ -n "${arg_restore_url}" ]]; then
-    set_progress "15" "Downloading restore data"
-
-    echo "Downloading backup: ${arg_restore_url} and key: ${arg_restore_key}"
-
-    while true; do
-        if $curl -L "${arg_restore_url}" | openssl aes-256-cbc -d -pass "pass:${arg_restore_key}" | tar -zxf - -C "${DATA_DIR}/box"; then break; fi
-        echo "Failed to download data, trying again"
-    done
-
-    set_progress "21" "Setting up MySQL"
-    if [[ -f "${DATA_DIR}/box/box.mysqldump" ]]; then
-        echo "Importing existing database into MySQL"
-        mysql -u root -p${mysql_root_password} box < "${DATA_DIR}/box/box.mysqldump"
-    fi
-fi
-
-set_progress "25" "Migrating data"
-sudo -u "${USER}" -H bash <<EOF
-set -eu
-cd "${BOX_SRC_DIR}"
-BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
 EOF
 
-set_progress "28" "Setup collectd"
-cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
-## collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
-#mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
-## detect device, let it fail if non exists
-#[[ -b "/dev/vda1" ]] && disk_device="/dev/vda1"
-#[[ -b "/dev/xvda1" ]] && disk_device="/dev/xvda1"
-#vda1_id=$(blkid -s UUID -o value ${disk_device})
-#ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
-service collectd restart
+set_progress "10" "Migrating data"
+sudo -u "${USER}" -H bash <<EOF
+set -eux
+cd "${box_src_tmp_dir}"
+PATH="${PATH}:${box_src_tmp_dir}/node_modules/.bin" npm run-script migrate_data
+EOF
 
-set_progress "30" "Setup nginx"
-mkdir -p "${DATA_DIR}/nginx/applications"
-cp "${script_dir}/start/nginx/nginx.conf" "${DATA_DIR}/nginx/nginx.conf"
-cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"
+set_progress "15" "Setup nginx"
+nginx_config_dir="${CONFIG_DIR}/nginx"
+nginx_appconfig_dir="${CONFIG_DIR}/nginx/applications"
 
-# generate these for update code paths as well to overwrite splash
-admin_cert_file="${DATA_DIR}/nginx/cert/host.cert"
-admin_key_file="${DATA_DIR}/nginx/cert/host.key"
-if [[ -f "${DATA_DIR}/box/certs/${admin_fqdn}.cert" && -f "${DATA_DIR}/box/certs/${admin_fqdn}.key" ]]; then
-    admin_cert_file="${DATA_DIR}/box/certs/${admin_fqdn}.cert"
-    admin_key_file="${DATA_DIR}/box/certs/${admin_fqdn}.key"
-fi
-${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+# copy nginx config
+mkdir -p "${nginx_appconfig_dir}"
+cp "${script_dir}/start/nginx/nginx.conf" "${nginx_config_dir}/nginx.conf"
+cp "${script_dir}/start/nginx/mime.types" "${nginx_config_dir}/mime.types"
+touch "${nginx_config_dir}/naked_domain.conf"
+sed -e "s/##ADMIN_FQDN##/${admin_fqdn}/" -e "s|##BOX_SRC_DIR##|${BOX_SRC_DIR}|" "${script_dir}/start/nginx/admin.conf_template" > "${nginx_appconfig_dir}/admin.conf"
 
-mkdir -p "${DATA_DIR}/nginx/cert"
-if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
-    cp "${DATA_DIR}/box/certs/host.cert" "${DATA_DIR}/nginx/cert/host.cert"
-    cp "${DATA_DIR}/box/certs/host.key" "${DATA_DIR}/nginx/cert/host.key"
-else
-    echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
-    echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
+certificate_dir="${nginx_config_dir}/cert"
+mkdir -p "${certificate_dir}"
+echo "${arg_tls_cert}" > ${certificate_dir}/host.cert
+echo "${arg_tls_key}" > ${certificate_dir}/host.key
+
+# link nginx config to system config
+unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
+ln -s "${nginx_config_dir}" /etc/nginx
+
+chown "${USER}:${USER}" -R "/home/${USER}"
+
+set_progress "20" "Removing existing container"
+# removing containers ensures containers are launched with latest config updates
+# restore code in appatask does not delete old containers
+existing_containers=$(docker ps -qa)
+echo "Remove containers: ${existing_containers}"
+if [[ -n "${existing_containers}" ]]; then
+    echo "${existing_containers}" | xargs docker rm -f
 fi
 
-set_progress "33" "Changing ownership"
-chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
-chown "${USER}:${USER}" "${DATA_DIR}"
+set_progress "30" "Setup collectd and graphite"
+${script_dir}/start/setup_collectd.sh
 
-set_progress "40" "Setting up infra"
-${script_dir}/start/setup_infra.sh "${arg_fqdn}"
+set_progress "40" "Setup haraka mail relay"
+docker rm -f haraka || true
+docker pull girish/haraka:0.1 || true # this line is for dev convenience since it's already part of base image
+haraka_container_id=$(docker run --restart=always -d --name="haraka" --cap-add="NET_ADMIN"\
+    -p 127.0.0.1:25:25 \
+    -h "${arg_fqdn}" \
+    -e "DOMAIN_NAME=${arg_fqdn}" \
+    -v "${CONFIG_DIR}/haraka:/app/data" \
+    girish/haraka:0.1)
+echo "Haraka container id: ${haraka_container_id}"
+# Every docker restart results in a new IP. Give our mail server a
+# static IP. Alternately, we need to link the mail container with
+# all our apps
+# This IP is set by the haraka container on every start and the firewall
+# allows connect to port 25. The ping gets the ARP lookup working
+echo "Checking connectivity to haraka(${MAIL_SERVER_IP})"
+if ! ping -c 20 "${MAIL_SERVER_IP}"; then
+    echo "Could not connect to mail server"
+fi
 
-set_progress "65" "Creating cloudron.conf"
+set_progress "50" "Setup MySQL addon"
+docker rm -f mysql || true
+mysql_root_password=$(pwgen -1 -s)
+docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
+docker pull girish/mysql:0.1 || true # this line for dev convenience since it's already part of base image
+mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -p 127.0.0.1:3306:3306 \
+    -h "${arg_fqdn}" \
+    -e "MYSQL_ROOT_PASSWORD=${mysql_root_password}" \
+    -e "MYSQL_ROOT_HOST=${docker0_ip}" \
+    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
+    girish/mysql:0.1)
+echo "MySQL container id: ${mysql_container_id}"
+
+set_progress "60" "Setup Postgres addon"
+docker rm -f postgresql || true
+postgresql_root_password=$(pwgen -1 -s)
+docker pull girish/postgresql:0.1 || true # this line for dev convenience since it's already part of base image
+postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -p 127.0.0.1:5432:5432 \
+    -h "${arg_fqdn}" \
+    -e "POSTGRESQL_ROOT_PASSWORD=${postgresql_root_password}" \
+    -v "${DATA_DIR}/postgresql:/var/lib/mysql" \
+    girish/postgresql:0.1)
+echo "PostgreSQL container id: ${postgresql_container_id}"
+
+set_progress "70" "Pulling Redis addon"
+docker pull girish/redis:0.1 || true # this line for dev convenience since it's already part of base image
+
+set_progress "80" "Creating cloudron.conf"
+cloudron_sqlite="${DATA_DIR}/cloudron.sqlite"
+admin_origin="https://${admin_fqdn}"
 sudo -u yellowtent -H bash <<EOF
-set -eu
+set -eux
 echo "Creating cloudron.conf"
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
     "token": "${arg_token}",
-    "apiServerOrigin": "${arg_api_server_origin}",
-    "webServerOrigin": "${arg_web_server_origin}",
+    "appServerUrl": "${arg_app_server_url}",
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "admin@${arg_fqdn}",
-    "provider": "${arg_provider}",
-    "database": {
-        "hostname": "localhost",
-        "username": "root",
-        "password": "${mysql_root_password}",
-        "port": 3306,
-        "name": "box"
+    "mailServer": "${MAIL_SERVER_IP}",
+    "mailUsername": "admin@${arg_fqdn}",
+    "addons": {
+        "mysql": {
+            "rootPassword": "${mysql_root_password}"
+        },
+        "postgresql": {
+            "rootPassword": "${postgresql_root_password}"
+        }
     }
 }
 CONF_END
 
-echo "Creating config.json for webadmin"
-cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
-{
-    "webServerOrigin": "${arg_web_server_origin}"
-}
-CONF_END
-EOF
-
-# Add Backup Configuration
-if [[ ! -z "${arg_backup_config}" ]]; then
-    echo "Add Backup Config"
-
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
-fi
-
-# Add DNS Configuration
-if [[ ! -z "${arg_dns_config}" ]]; then
-    echo "Add DNS Config"
-
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
-fi
-
-# Add TLS Configuration
-if [[ ! -z "${arg_tls_config}" ]]; then
-    echo "Add TLS Config"
-
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
-fi
+echo "Marking apps for restore"
+# TODO: do not auto-start stopped containers (httpPort might need fixing to start them)
+sqlite3 "${cloudron_sqlite}" 'UPDATE apps SET installationState = "pending_restore", healthy = NULL, runState = NULL, containerId = NULL, httpPort = NULL, installationProgress = NULL'
 
 # Add webadmin oauth client
-# The domain might have changed, therefor we have to update the record
-# !!! This needs to be in sync with the webadmin, specifically login_callback.js
 echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
-mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+ADMIN_SCOPES="root,profile,users,apps,settings,roleAdmin"
+ADMIN_ID=$(cat /proc/sys/kernel/random/uuid)
+sqlite3 "${cloudron_sqlite}" "INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES (\"\$ADMIN_ID\", \"webadmin\", \"cid-webadmin\", \"secret-webadmin\", \"WebAdmin\", \"${admin_origin}\", \"\$ADMIN_SCOPES\")"
 
-echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
-mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+EOF
 
-set_progress "80" "Starting Cloudron"
-systemctl start cloudron.target
+# bookkeep the version as part of data
+echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/version"
 
-sleep 2 # give systemd sometime to start the processes
+set_progress "90" "Setup supervisord"
+${script_dir}/start/setup_supervisord.sh
 
-set_progress "85" "Reloading nginx"
+set_progress "95" "Reloading supervisor"
+${script_dir}/start/reload_supervisord.sh
+
+set_progress "99" "Reloading nginx"
 nginx -s reload
 
 set_progress "100" "Done"

setup/start/collectd/collectd.conf

@@ -52,20 +52,20 @@ Interval 20
 # accessed.                                                                  #
 ##############################################################################
 
-LoadPlugin logfile
-#LoadPlugin syslog
+#LoadPlugin logfile
+LoadPlugin syslog
 
-<Plugin logfile>
-   LogLevel "info"
-   File "/var/log/collectd.log"
-   Timestamp true
-   PrintSeverity false
-</Plugin>
-
-#<Plugin syslog>
-#    LogLevel info
+#<Plugin logfile>
+#   LogLevel "info"
+#   File STDOUT
+#   Timestamp true
+#   PrintSeverity false
 #</Plugin>
 
+<Plugin syslog>
+    LogLevel info
+</Plugin>
+
 ##############################################################################
 # LoadPlugin section                                                         #
 #----------------------------------------------------------------------------#
@@ -96,7 +96,7 @@ LoadPlugin df
 #LoadPlugin entropy
 #LoadPlugin ethstat
 #LoadPlugin exec
-#LoadPlugin filecount
+LoadPlugin filecount
 #LoadPlugin fscache
 #LoadPlugin gmond
 #LoadPlugin hddtemp
@@ -193,11 +193,12 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin df>
-   FSType "tmpfs"
-   MountPoint "/dev"
+   Device "/dev/vda1"
+   Device "/dev/loop0"
+   Device "/dev/loop1"
 
    ReportByDevice true
-   IgnoreSelected true
+   IgnoreSelected false
 
    ValuesAbsolute true
    ValuesPercentage true
@@ -220,7 +221,7 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin processes>
-   ProcessMatch "app" "node box.js"
+   ProcessMatch "app" "node app.js"
 </Plugin>
 
 <Plugin swap>

setup/start/nginx/admin.conf_template

@@ -0,0 +1,33 @@
+server {
+    listen       443;
+    server_name  ##ADMIN_FQDN##;
+
+    ssl                  on;
+    # paths are relative to prefix and not to this file
+    ssl_certificate      cert/host.cert;
+    ssl_certificate_key  cert/host.key;
+
+
+    ssl_session_timeout  5m;
+
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+    ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
+    ssl_prefer_server_ciphers on;
+
+    location /api/ {
+        proxy_pass   http://127.0.0.1:3000;
+        client_max_body_size 1m;
+    }
+
+    # graphite paths
+    location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+        proxy_pass   http://127.0.0.1:8000;
+        client_max_body_size 1m;
+    }
+
+    location / {
+        root   ##BOX_SRC_DIR##/webadmin/dist/;
+        index  index.html index.htm;
+    }
+}
+

setup/start/nginx/appconfig.ejs

@@ -1,114 +0,0 @@
-# http://nginx.org/en/docs/http/websocket.html
-map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-}
-
-server {
-    listen       443;
-    server_name  <%= vhost %>;
-
-    ssl                  on;
-    # paths are relative to prefix and not to this file
-    ssl_certificate      <%= certFilePath %>;
-    ssl_certificate_key  <%= keyFilePath %>;
-    ssl_session_timeout  5m;
-    ssl_session_cache shared:SSL:50m;
-
-    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
-    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
-    # https://cipherli.st/
-    ssl_prefer_server_ciphers on;
-    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers 'AES128+EECDH:AES128+EDH';
-    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
-
-    proxy_http_version 1.1;
-    proxy_intercept_errors on;
-    proxy_read_timeout       3500;
-    proxy_connect_timeout    3250;
-
-    proxy_set_header   Host               $host;
-    proxy_set_header   X-Forwarded-For    $remote_addr;
-    proxy_set_header   X-Forwarded-Host   $host;
-    proxy_set_header   X-Forwarded-Proto  https;
-
-    # upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
-    proxy_set_header Upgrade $http_upgrade;
-    proxy_set_header Connection $connection_upgrade;
-
-    # only serve up the status page if we get proxy gateway errors
-    error_page 502 503 504 @appstatus;
-    location @appstatus {
-        return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
-    }
-
-    location / {
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # Disable check to allow unlimited body sizes
-        client_max_body_size 0;
-
-<% if ( endpoint === 'admin' ) { %>
-        location /api/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 1m;
-        }
-
-        # graphite paths
-        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
-            proxy_pass   http://127.0.0.1:8000;
-            client_max_body_size 1m;
-        }
-
-        location / {
-            root   <%= sourceDir %>/webadmin/dist;
-            index  index.html index.htm;
-        }
-
-<% } else if ( endpoint === 'oauthproxy' ) { %>
-        proxy_pass http://127.0.0.1:3003;
-        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
-<% } else if ( endpoint === 'app' ) { %>
-        proxy_pass http://127.0.0.1:<%= port %>;
-<% } else if ( endpoint === 'splash' ) { %>
-        root   <%= sourceDir %>;
-
-        error_page 503 /update.html;
-
-        location /update.html {
-            add_header Cache-Control no-cache;
-        }
-
-        location /theme.css {
-            add_header Cache-Control no-cache;
-        }
-
-        location /3rdparty/ {
-            add_header Cache-Control no-cache;
-        }
-
-        location /js/ {
-            add_header Cache-Control no-cache;
-        }
-
-        location /progress.json {
-            add_header Cache-Control no-cache;
-        }
-
-        location /api/v1/cloudron/progress {
-            add_header Cache-Control no-cache;
-            default_type application/json;
-            alias <%= sourceDir %>/progress.json;
-        }
-
-        location / {
-            return 503;
-        }
-<% } %>
-    }
-}
-

setup/start/nginx/nginx.conf

@@ -17,9 +17,6 @@ http {
         '"$request" $status $body_bytes_sent $request_time '
         '"$http_referer" "$http_user_agent"';
 
-    # required for long host names
-    server_names_hash_bucket_size 128;
-
     access_log access.log combined2;
 
     sendfile        on;
@@ -38,12 +35,6 @@ http {
             deny all;
         }
 
-        # acme challenges
-        location /.well-known/acme-challenge/ {
-            default_type text/plain;
-            alias /home/yellowtent/data/acme/;
-        }
-
         location / {
             # redirect everything to HTTPS
             return 301 https://$host$request_uri;
@@ -58,16 +49,11 @@ http {
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
-        error_page 404 = @fallback;
-        location @fallback {
-            internal;
-            root /home/yellowtent/box/webadmin/dist;
-            rewrite ^/$ /nakeddomain.html break;
-        }
-
         return 404;
     }
 
+    include naked_domain.conf;
+
     include applications/*.conf;
 }
 

setup/start/reload_supervisord.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+
+# looks like restarting supervisor completely is the only way to reload it
+service supervisor stop || true
+
+echo -n "Waiting for supervisord to stop"
+while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
+    echo -n "."
+    sleep 1
+done
+echo ""
+
+echo "Starting supervisor"
+
+service supervisor start
+
+sleep 2 # give supervisor sometime to start the processes
+

setup/start/setup_collectd.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -eu
+
+readonly GRAPHITE_DIR="/home/yellowtent/data/graphite"
+readonly COLLECTD_CONFIG_DIR="/home/yellowtent/configs/collectd"
+readonly COLLECTD_APPCONFIG_DIR="${COLLECTD_CONFIG_DIR}/collectd.conf.d"
+
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+mkdir -p "${GRAPHITE_DIR}"
+
+docker rm -f graphite || true
+docker pull girish/graphite:0.2
+docker run --restart=always -d --name="graphite" \
+    -p 127.0.0.1:2003:2003 \
+    -p 127.0.0.1:2004:2004 \
+    -p 127.0.0.1:8000:8000 \
+    -v "${GRAPHITE_DIR}:/app/data" girish/graphite:0.2
+
+mkdir -p "${COLLECTD_APPCONFIG_DIR}"
+cp -r "${script_dir}/collectd/collectd.conf" "${COLLECTD_CONFIG_DIR}/collectd.conf"
+rm -rf /etc/collectd
+ln -sfF "${COLLECTD_CONFIG_DIR}" /etc/collectd
+chown -R yellowtent.yellowtent "${COLLECTD_CONFIG_DIR}"
+
+update-rc.d -f collectd defaults
+/etc/init.d/collectd restart
+

setup/start/setup_infra.sh

@@ -1,129 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-arg_fqdn="$1"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${arg_fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"

setup/start/setup_supervisord.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -eu
+
+readonly BOX_SRC_DIR="/home/yellowtent/box"
+readonly DATA_DIR="/home/yellowtent/data"
+
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+rm -rf /etc/supervisor
+mkdir -p /etc/supervisor/conf.d
+cp "${script_dir}/supervisord/supervisord.conf" /etc/supervisor/
+
+echo "Writing supervisor configs..."
+
+cat > /etc/supervisor/conf.d/box.conf <<EOF
+[program:box]
+command=/usr/bin/node "${BOX_SRC_DIR}/app.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/box.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",NODE_ENV="cloudron"
+EOF
+
+cat > /etc/supervisor/conf.d/oauthproxy.conf <<EOF
+[program:oauthproxy]
+command=/usr/bin/node "${BOX_SRC_DIR}/oauthproxy.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/proxy.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
+EOF
+
+cat > /etc/supervisor/conf.d/apphealthtask.conf <<EOF
+[program:apphealthtask]
+command=/usr/bin/node "${BOX_SRC_DIR}/apphealthtask.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/apphealthtask.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
+EOF
+

setup/start/supervisord/supervisord.conf

@@ -0,0 +1,33 @@
+; supervisor config file
+
+; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+nodaemon = false
+childlogdir = /var/log/supervisor/
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=http://127.0.0.1:9001
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = conf.d/*.conf
+

setup/stop.sh

@@ -1,7 +1,15 @@
 #!/bin/bash
 
-set -eu -o pipefail
+set -eu
 
-echo "Stopping cloudron"
+echo "Stopping box code"
+
+service supervisor stop || true
+
+echo -n "Waiting for supervisord to stop"
+while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
+    echo -n "."
+    sleep 1
+done
+echo ""
 
-systemctl stop cloudron.target

src/appdb.js

@@ -2,17 +2,25 @@
 
 'use strict';
 
+var assert = require('assert'),
+    async = require('async'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    debug = require('debug')('box:appdb'),
+    safe = require('safetydance'),
+    util = require('util');
+
 exports = module.exports = {
     get: get,
     getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
-    getByContainerId: getByContainerId,
     add: add,
     exists: exists,
     del: del,
     update: update,
     getAll: getAll,
     getPortBindings: getPortBindings,
+    clear: clear,
 
     setAddonConfig: setAddonConfig,
     getAddonConfig: getAddonConfig,
@@ -23,220 +31,179 @@ exports = module.exports = {
     setHealth: setHealth,
     setInstallationCommand: setInstallationCommand,
     setRunCommand: setRunCommand,
-    getAppStoreIds: getAppStoreIds,
+    getAppVersions: getAppVersions,
 
-    // installation codes (keep in sync in UI)
-    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
-    ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
-    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
-    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
-    ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
-    ISTATE_PENDING_FORCE_UPDATE: 'pending_force_update', // update from any state preserving data
-    ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app
-    ISTATE_ERROR: 'error', // error executing last pending_* command
-    ISTATE_INSTALLED: 'installed', // app is installed
+    // status codes
+    ISTATE_PENDING_INSTALL: 'pending_install',
+    ISTATE_PENDING_CONFIGURE: 'pending_configure',
+    ISTATE_PENDING_UNINSTALL: 'pending_uninstall',
+    ISTATE_PENDING_RESTORE: 'pending_restore',
+    ISTATE_PENDING_UPDATE: 'pending_update',
+    ISTATE_ERROR: 'error',
+    ISTATE_INSTALLED: 'installed',
 
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
-    RSTATE_STOPPED: 'stopped', // app stopped by use
-
-    // run codes (keep in sync in UI)
-    HEALTH_HEALTHY: 'healthy',
-    HEALTH_UNHEALTHY: 'unhealthy',
-    HEALTH_ERROR: 'error',
-    HEALTH_DEAD: 'dead',
-
-    _clear: clear
+    RSTATE_STOPPED: 'stopped', // app stopped by user
+    RSTATE_DEAD: 'dead', // app stopped on it's own
+    RSTATE_ERROR: 'error'
 };
 
-var assert = require('assert'),
-    async = require('async'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror'),
-    safe = require('safetydance'),
-    util = require('util');
+var APPS_FIELDS = [ 'id', 'appStoreId', 'version', 'installationState', 'installationProgress', 'runState',
+    'healthy', 'containerId', 'manifestJson', 'httpPort', 'location', 'dnsRecordId', 'accessRestriction' ].join(',');
 
-var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
-    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
+var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.version', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
+    'apps.healthy', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId', 'apps.accessRestriction' ].join(',');
 
-var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
+var PORT_BINDINGS_FIELDS = [ 'hostPort', 'containerPort', 'appId' ].join(',');
 
 function postProcess(result) {
-    assert.strictEqual(typeof result, 'object');
-
     assert(result.manifestJson === null || typeof result.manifestJson === 'string');
+
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
-    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
-    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
-    delete result.lastBackupConfigJson;
-
-    assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
-    result.oldConfig = safe.JSON.parse(result.oldConfigJson);
-    delete result.oldConfigJson;
-
     assert(result.hostPorts === null || typeof result.hostPorts === 'string');
-    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
+    assert(result.containerPorts === null || typeof result.containerPorts === 'string');
 
     result.portBindings = { };
     var hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
-    var environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
+    var containerPorts = result.containerPorts === null ? [ ] : result.containerPorts.split(',');
 
     delete result.hostPorts;
-    delete result.environmentVariables;
+    delete result.containerPorts;
 
-    for (var i = 0; i < environmentVariables.length; i++) {
-        result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
+    for (var i = 0; i < hostPorts.length; i++) {
+        result.portBindings[containerPorts[i]] = hostPorts[i];
     }
-
-    result.oauthProxy = !!result.oauthProxy;
-
-    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
-    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
-    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
-    delete result.accessRestrictionJson;
 }
 
 function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
+    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
+        + ' FROM apps LEFT OUTER JOIN appPortBindings WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        postProcess(result[0]);
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, result[0]);
+        postProcess(result);
+
+        callback(null, result);
     });
 }
 
 function getBySubdomain(subdomain, callback) {
-    assert.strictEqual(typeof subdomain, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof subdomain === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
+    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        postProcess(result[0]);
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, result[0]);
+        postProcess(result);
+
+        callback(null, result);
     });
 }
 
 function getByHttpPort(httpPort, callback) {
-    assert.strictEqual(typeof httpPort, 'number');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof httpPort === 'number');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
+    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        postProcess(result[0]);
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, result[0]);
-    });
-}
+        postProcess(result);
 
-function getByContainerId(containerId, callback) {
-    assert.strictEqual(typeof containerId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        postProcess(result[0]);
-
-        callback(null, result[0]);
+        callback(null, result);
     });
 }
 
 function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+    database.all('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
         + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
         + ' GROUP BY apps.id ORDER BY apps.id', function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
+        if (typeof results === 'undefined') results = [ ];
+
         results.forEach(postProcess);
 
         callback(null, results);
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof manifest.version, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof oauthProxy, 'boolean');
-    assert.strictEqual(typeof callback, 'function');
+function add(id, appStoreId, location, portBindings, accessRestriction, callback) {
+    assert(typeof id === 'string');
+    assert(typeof appStoreId === 'string');
+    assert(typeof location === 'string');
+    assert(typeof portBindings === 'object');
+    assert(typeof accessRestriction === 'string');
+    assert(typeof callback === 'function');
 
     portBindings = portBindings || { };
 
-    var manifestJson = JSON.stringify(manifest);
-    var accessRestrictionJson = JSON.stringify(accessRestriction);
+    var conn = database.beginTransaction();
 
-    var queries = [ ];
-    queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
-    });
+    conn.run('INSERT INTO apps (id, appStoreId, installationState, location, accessRestriction) VALUES (?, ?, ?, ?, ?)',
+           [ id, appStoreId, exports.ISTATE_PENDING_INSTALL, location, accessRestriction ], function (error) {
+        if (error || !this.lastID) database.rollback(conn);
 
-    Object.keys(portBindings).forEach(function (env) {
-        queries.push({
-            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, appId) VALUES (?, ?, ?)',
-            args: [ env, portBindings[env], id ]
+        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+
+        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        async.eachSeries(Object.keys(portBindings), function iterator(containerPort, callback) {
+            conn.run('INSERT INTO appPortBindings (hostPort, containerPort, appId) VALUES (?, ?, ?)',
+                     [ portBindings[containerPort], containerPort, id ], callback);
+        }, function done(error) {
+            if (error) database.rollback(conn);
+
+            if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+
+            if (error /* || !this.lastID*/) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            database.commit(conn, callback);
         });
     });
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        callback(null);
-    });
 }
 
 function exists(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
+    database.get('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null, result.length !== 0);
+        return callback(null, typeof result !== 'undefined');
     });
 }
 
 function getPortBindings(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
+    database.all('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
+        results = results || [ ];
         var portBindings = { };
         for (var i = 0; i < results.length; i++) {
-            portBindings[results[i].environmentVariable] = results[i].hostPort;
+            portBindings[results[i].containerPort] = results[i].hostPort;
         }
 
         callback(null, portBindings);
@@ -244,29 +211,29 @@ function getPortBindings(id, callback) {
 }
 
 function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof callback === 'function');
 
-    var queries = [
-        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
-        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
-    ];
+    var conn = database.beginTransaction();
+    conn.run('DELETE FROM appPortBindings WHERE appId = ?', [ id ], function (error) {
+        conn.run('DELETE FROM apps WHERE id = ?', [ id ], function (error) {
+            if (error || this.changes !== 1) database.rollback(conn);
 
-    database.transaction(queries, function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null);
+            database.commit(conn, callback);
+        });
     });
 }
 
 function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
     async.series([
-        database.query.bind(null, 'DELETE FROM appPortBindings'),
-        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
-        database.query.bind(null, 'DELETE FROM apps')
+        database.run.bind(null, 'DELETE FROM appPortBindings'),
+        database.run.bind(null, 'DELETE FROM apps'),
+        database.run.bind(null, 'DELETE FROM appAddonConfigs')
     ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         return callback(null);
@@ -274,158 +241,150 @@ function clear(callback) {
 }
 
 function update(id, app, callback) {
-    updateWithConstraints(id, app, '', callback);
+    updateWithConstraints(id, app, callback);
 }
 
 function updateWithConstraints(id, app, constraints, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof constraints, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof app === 'object');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
 
-    var queries = [ ];
-
-    if ('portBindings' in app) {
-        var portBindings = app.portBindings || { };
-        // replace entries by app id
-        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
-        Object.keys(portBindings).forEach(function (env) {
-            var values = [ portBindings[env], env, id ];
-            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, environmentVariable, appId) VALUES(?, ?, ?)', args: values });
-        });
+    if (typeof constraints === 'function') {
+        callback = constraints;
+        constraints = '';
+    } else {
+        assert(typeof constraints === 'string');
+        assert(typeof callback === 'function');
     }
 
-    var fields = [ ], values = [ ];
-    for (var p in app) {
-        if (p === 'manifest') {
-            fields.push('manifestJson = ?');
-            values.push(JSON.stringify(app[p]));
-        } else if (p === 'lastBackupConfig') {
-            fields.push('lastBackupConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
-        } else if (p === 'oldConfig') {
-            fields.push('oldConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
-        } else if (p === 'accessRestriction') {
-            fields.push('accessRestrictionJson = ?');
-            values.push(JSON.stringify(app[p]));
-        } else if (p !== 'portBindings') {
-            fields.push(p + ' = ?');
-            values.push(app[p]);
+    var portBindings = app.portBindings || { };
+
+    var conn = database.beginTransaction();
+    async.eachSeries(Object.keys(portBindings), function iterator(containerPort, callback) {
+        var values = [ portBindings[containerPort], containerPort, id ];
+        conn.run('UPDATE appPortBindings SET hostPort = ? WHERE containerPort = ? AND appId = ?', values, callback);
+    }, function seriesDone(error) {
+        if (error) {
+            database.rollback(conn);
+            return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         }
-    }
 
-    if (values.length !== 0) {
+        var args = [ ], values = [ ];
+        for (var p in app) {
+            if (!app.hasOwnProperty(p)) continue;
+
+            if (p === 'manifest') {
+                args.push('manifestJson = ?');
+                values.push(JSON.stringify(app[p]));
+            } else if (p !== 'portBindings') {
+                args.push(p + ' = ?');
+                values.push(app[p]);
+            }
+        }
+
+        if (values.length === 0) return database.commit(conn, callback);
+
         values.push(id);
-        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
-    }
 
-    database.transaction(queries, function (error, results) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results[results.length - 1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        conn.run('UPDATE apps SET ' + args.join(', ') + ' WHERE id = ? ' + constraints, values, function (error) {
+            if (error || this.changes !== 1) database.rollback(conn);
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+            database.commit(conn, callback);
+        });
     });
 }
 
-// not sure if health should influence runState
-function setHealth(appId, health, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof health, 'string');
-    assert.strictEqual(typeof callback, 'function');
+// sets health on installed apps that have a runState which is not null or pending
+function setHealth(appId, healthy, runState, callback) {
+    assert(typeof appId === 'string');
+    assert(typeof healthy === 'boolean');
+    assert(typeof runState === 'string');
+    assert(typeof callback === 'function');
 
-    var values = { health: health };
+    var values = {
+        healthy: healthy,
+        runState: runState
+    };
 
-    var constraints = 'AND runState NOT LIKE "pending_%" AND installationState = "installed"';
+    var constraints = 'AND runState NOT GLOB "pending_*" AND installationState = "installed"';
+    if (runState === exports.RSTATE_DEAD) { // don't mark stopped apps as dead
+        constraints += ' AND runState != "stopped"';
+    }
 
     updateWithConstraints(appId, values, constraints, callback);
 }
 
 function setInstallationCommand(appId, installationState, values, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof installationState, 'string');
+    assert(typeof appId === 'string');
+    assert(typeof installationState === 'string');
 
     if (typeof values === 'function') {
         callback = values;
         values = { };
     } else {
-        assert.strictEqual(typeof values, 'object');
-        assert.strictEqual(typeof callback, 'function');
+        assert(typeof values === 'object');
+        assert(typeof callback === 'function');
     }
 
     values.installationState = installationState;
-    values.installationProgress = '';
 
-    // Rules are:
-    // uninstall is allowed in any state
-    // force update is allowed in any state including pending_uninstall! (for better or worse)
-    // restore is allowed from installed or error state
-    // update and configure are allowed only in installed state
-
-    if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
+    if (installationState === exports.ISTATE_PENDING_UNINSTALL) {
         updateWithConstraints(appId, values, '', callback);
-    } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
-        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
-        updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
     } else {
-        callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
+        updateWithConstraints(appId, values, 'AND installationState NOT GLOB "pending_*"', callback);
     }
 }
 
 function setRunCommand(appId, runState, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof runState, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof runState === 'string');
+    assert(typeof callback === 'function');
 
     var values = { runState: runState };
-    updateWithConstraints(appId, values, 'AND runState NOT LIKE "pending_%" AND installationState = "installed"', callback);
+    updateWithConstraints(appId, values, 'AND runState NOT GLOB "pending_*" AND installationState = "installed"', callback);
 }
 
-function getAppStoreIds(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function getAppVersions(callback) {
+    assert(typeof callback === 'function');
 
-    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
+    database.all('SELECT id, appStoreId, version FROM apps', function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
+        results = results || [ ];
         callback(null, results);
     });
 }
 
 function setAddonConfig(appId, addonId, env, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
+    assert(typeof appId === 'string');
+    assert(typeof addonId === 'string');
     assert(util.isArray(env));
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    unsetAddonConfig(appId, addonId, function (error) {
-        if (error) return callback(error);
+    if (env.length === 0) return callback(null);
 
-        if (env.length === 0) return callback(null);
+    var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
+    var args = [ ], queryArgs = [ ];
+    for (var i = 0; i < env.length; i++) {
+        args.push(appId, addonId, env[i]);
+        queryArgs.push('(?, ?, ?)');
+    }
 
-        var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
-        var args = [ ], queryArgs = [ ];
-        for (var i = 0; i < env.length; i++) {
-            args.push(appId, addonId, env[i]);
-            queryArgs.push('(?, ?, ?)');
-        }
+    database.run(query + queryArgs.join(','), args, function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        database.query(query + queryArgs.join(','), args, function (error) {
-            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-            return callback(null);
-        });
+        return callback(null);
     });
 }
 
 function unsetAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof addonId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
+    database.run('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -433,10 +392,10 @@ function unsetAddonConfig(appId, addonId, callback) {
 }
 
 function unsetAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
+    database.run('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -444,29 +403,29 @@ function unsetAddonConfigByAppId(appId, callback) {
 }
 
 function getAddonConfig(appId, addonId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof addonId, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof addonId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
+    database.all('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         var config = [ ];
-        results.forEach(function (v) { config.push(v.value); });
+        result.forEach(function (v) { config.push(v.value); });
 
         callback(null, config);
     });
 }
 
 function getAddonConfigByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
+    database.all('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         var config = [ ];
-        results.forEach(function (v) { config.push(v.value); });
+        result.forEach(function (v) { config.push(v.value); });
 
         callback(null, config);
     });

src/apphealthmonitor.js

@@ -1,195 +0,0 @@
-'use strict';
-
-var appdb = require('./appdb.js'),
-    assert = require('assert'),
-    async = require('async'),
-    DatabaseError = require('./databaseerror.js'),
-    debug = require('debug')('box:apphealthmonitor'),
-    docker = require('./docker.js').connection,
-    mailer = require('./mailer.js'),
-    superagent = require('superagent'),
-    util = require('util');
-
-exports = module.exports = {
-    start: start,
-    stop: stop
-};
-
-var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
-var gHealthInfo = { }; // { time, emailSent }
-var gRunTimeout = null;
-var gDockerEventStream = null;
-
-function debugApp(app) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function setHealth(app, health, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof health, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var now = new Date();
-
-    if (!(app.id in gHealthInfo)) { // add new apps to list
-        gHealthInfo[app.id] = { time: now, emailSent: false };
-    }
-
-    if (health === appdb.HEALTH_HEALTHY) {
-        gHealthInfo[app.id].time = now;
-    } else if (Math.abs(now - gHealthInfo[app.id].time) > UNHEALTHY_THRESHOLD) {
-        if (gHealthInfo[app.id].emailSent) return callback(null);
-
-        debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
-
-        if (app.appStoreId !== '') mailer.appDied(app); // do not send mails for dev apps
-        gHealthInfo[app.id].emailSent = true;
-    } else {
-        debugApp(app, 'waiting for sometime to update the app health');
-        return callback(null);
-    }
-
-    appdb.setHealth(app.id, health, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled?
-        if (error) return callback(error);
-
-        app.health = health;
-
-        callback(null);
-    });
-}
-
-
-// callback is called with error for fatal errors and not if health check failed
-function checkAppHealth(app, callback) {
-    if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
-        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
-        return callback(null);
-    }
-
-    var container = docker.getContainer(app.containerId),
-        manifest = app.manifest;
-
-    container.inspect(function (err, data) {
-        if (err || !data || !data.State) {
-            debugApp(app, 'Error inspecting container');
-            return setHealth(app, appdb.HEALTH_ERROR, callback);
-        }
-
-        if (data.State.Running !== true) {
-            debugApp(app, 'exited');
-            return setHealth(app, appdb.HEALTH_DEAD, callback);
-        }
-
-        // poll through docker network instead of nginx to bypass any potential oauth proxy
-        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
-        superagent
-            .get(healthCheckUrl)
-            .redirects(0)
-            .timeout(HEALTHCHECK_INTERVAL)
-            .end(function (error, res) {
-            if (error && !error.response) {
-                debugApp(app, 'not alive (network error): %s', error.message);
-                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
-            } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
-                debugApp(app, 'not alive : %s', error || res.status);
-                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
-            } else {
-                setHealth(app, appdb.HEALTH_HEALTHY, callback);
-            }
-        });
-    });
-}
-
-function processApps(callback) {
-    appdb.getAll(function (error, apps) {
-        if (error) return callback(error);
-
-        async.each(apps, checkAppHealth, function (error) {
-            if (error) console.error(error);
-
-            var alive = apps
-                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
-                .map(function (a) { return a.location; }).join(', ');
-
-            debug('apps alive: [%s]', alive);
-
-            callback(null);
-        });
-    });
-}
-
-function run() {
-    processApps(function (error) {
-        if (error) console.error(error);
-
-        gRunTimeout = setTimeout(run, HEALTHCHECK_INTERVAL);
-    });
-}
-
-/*
-    OOM can be tested using stress tool like so:
-        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
-        apt-get update && apt-get install stress
-        stress --vm 1 --vm-bytes 200M --vm-hang 0
-*/
-function processDockerEvents() {
-    // note that for some reason, the callback is called only on the first event
-    debug('Listening for docker events');
-    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
-        if (error) return console.error(error);
-
-        gDockerEventStream = stream;
-
-        stream.setEncoding('utf8');
-        stream.on('data', function (data) {
-            var ev = JSON.parse(data);
-            debug('Container ' + ev.id + ' went OOM');
-            appdb.getByContainerId(ev.id, function (error, app) {
-                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
-                var context = JSON.stringify(ev);
-                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
-
-                debug('OOM Context: %s', context);
-
-                // do not send mails for dev apps
-                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
-            });
-        });
-
-        stream.on('error', function (error) {
-            console.error('Error reading docker events', error);
-            gDockerEventStream = null; // TODO: reconnect?
-        });
-
-        stream.on('end', function () {
-            console.error('Docker event stream ended');
-            gDockerEventStream = null; // TODO: reconnect?
-        });
-    });
-}
-
-function start(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Starting apphealthmonitor');
-
-    processDockerEvents();
-
-    run();
-
-    callback();
-}
-
-function stop(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    clearTimeout(gRunTimeout);
-    gDockerEventStream.end();
-
-    callback();
-}

src/auth.js

@@ -2,16 +2,12 @@
 
 'use strict';
 
-exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize
-};
-
 var assert = require('assert'),
     BasicStrategy = require('passport-http').BasicStrategy,
     BearerStrategy = require('passport-http-bearer').Strategy,
     clientdb = require('./clientdb'),
     ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    database = require('./database'),
     DatabaseError = require('./databaseerror'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
@@ -20,11 +16,15 @@ var assert = require('assert'),
     tokendb = require('./tokendb'),
     user = require('./user'),
     userdb = require('./userdb'),
-    UserError = user.UserError,
-    _ = require('underscore');
+    UserError = user.UserError;
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize
+};
 
 function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
     passport.serializeUser(function (user, callback) {
         callback(null, user.username);
@@ -42,31 +42,21 @@ function initialize(callback) {
     });
 
     passport.use(new LocalStrategy(function (username, password, callback) {
-        if (username.indexOf('@') === -1) {
-            user.verify(username, password, function (error, result) {
-                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, _.pick(result, 'id', 'username', 'email', 'admin'));
-            });
-        } else {
-            user.verifyWithEmail(username, password, function (error, result) {
-                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
-                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
-                if (error) return callback(error);
-                if (!result) return callback(null, false);
-                callback(null, _.pick(result, 'id', 'username', 'email', 'admin'));
-            });
-        }
+        user.verify(username, password, function (error, result) {
+            if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+            if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
+            if (error) return callback(error);
+            if (!result) return callback(null, false);
+            callback(null, database.removePrivates(result));
+        });
     }));
 
     passport.use(new BasicStrategy(function (username, password, callback) {
         if (username.indexOf('cid-') === 0) {
-            debug('BasicStrategy: detected client id %s instead of username:password', username);
+            debug('BasicStrategy: detected clientId %s instead of username:password', username);
             // username is actually client id here
             // password is client secret
-            clientdb.get(username, function (error, client) {
+            clientdb.getByClientId(username, function (error, client) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
                 if (client.clientSecret != password) return callback(null, false);
@@ -84,7 +74,7 @@ function initialize(callback) {
     }));
 
     passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
+        clientdb.getByClientId(clientId, function(error, client) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
             if (error) { return callback(error); }
             if (client.clientSecret != clientSecret) { return callback(null, false); }
@@ -97,32 +87,13 @@ function initialize(callback) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
-            // scopes here can define what capabilities that token carries
-            // passport put the 'info' object into req.authInfo, where we can further validate the scopes
-            var info = { scope: token.scope };
-            var tokenType;
-
-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }
-
-            userdb.get(token.identifier, function (error, user) {
+            userdb.get(token.userId, function (error, user) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
 
-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
+                // scopes here can define what capabilities that token carries
+                // passport put the 'info' object into req.authInfo, where we can further validate the scopes
+                var info = { scope: token.scope };
                 callback(null, user, info);
             });
         });
@@ -132,7 +103,7 @@ function initialize(callback) {
 }
 
 function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
     callback(null);
 }

src/authcodedb.js

@@ -2,74 +2,64 @@
 
 'use strict';
 
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    debug = require('debug')('box:authcodedb');
+
 exports = module.exports = {
     get: get,
     add: add,
     del: del,
-    delExpired: delExpired,
-
-    _clear: clear
+    clear: clear
 };
 
-var assert = require('assert'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror');
-
-var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId', 'expiresAt' ].join(',');
+var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId' ].join(',');
 
 function get(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof authCode === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ? AND expiresAt > ?', [ authCode, Date.now() ], function (error, result) {
+    database.get('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, result[0]);
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result);
     });
 }
 
-function add(authCode, clientId, userId, expiresAt, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof expiresAt, 'number');
-    assert.strictEqual(typeof callback, 'function');
+function add(authCode, clientId, userId, callback) {
+    assert(typeof authCode === 'string');
+    assert(typeof clientId === 'string');
+    assert(typeof userId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('INSERT INTO authcodes (authCode, clientId, userId, expiresAt) VALUES (?, ?, ?, ?)',
-            [ authCode, clientId, userId, expiresAt ], function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.run('INSERT INTO authcodes (authCode, clientId, userId) VALUES (?, ?, ?)',
+            [ authCode, clientId, userId ], function (error) {
+        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
     });
 }
 
 function del(authCode, callback) {
-    assert.strictEqual(typeof authCode, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof authCode === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
+    database.run('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         callback(null);
     });
 }
 
-function delExpired(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('DELETE FROM authcodes WHERE expiresAt <= ?', [ Date.now() ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        return callback(null, result.affectedRows);
-    });
-}
-
 function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM authcodes', function (error) {
+    database.run('DELETE FROM authcodes', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);

src/backups.js

@@ -1,142 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    BackupsError: BackupsError,
-
-    getAllPaged: getAllPaged,
-
-    getBackupUrl: getBackupUrl,
-    getRestoreUrl: getRestoreUrl,
-
-    copyLastBackup: copyLastBackup
-};
-
-var assert = require('assert'),
-    caas = require('./storage/caas.js'),
-    config = require('./config.js'),
-    debug = require('debug')('box:backups'),
-    s3 = require('./storage/s3.js'),
-    settings = require('./settings.js'),
-    util = require('util');
-
-function BackupsError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
-        this.message = reason;
-    } else if (typeof errorOrMessage === 'string') {
-        this.message = errorOrMessage;
-    } else {
-        this.message = 'Internal error';
-        this.nestedError = errorOrMessage;
-    }
-}
-util.inherits(BackupsError, Error);
-BackupsError.EXTERNAL_ERROR = 'external error';
-BackupsError.INTERNAL_ERROR = 'internal error';
-BackupsError.MISSING_CREDENTIALS = 'missing credentials';
-
-// choose which storage backend we use for test purpose we use s3
-function api(provider) {
-    switch (provider) {
-        case 'caas': return caas;
-        case 's3': return s3;
-        default: return null;
-    }
-}
-
-function getAllPaged(page, perPage, callback) {
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-        api(backupConfig.provider).getAllPaged(backupConfig, page, perPage, function (error, backups) {
-            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-
-            return callback(null, backups); // [ { creationTime, restoreKey } ] sorted by time (latest first
-        });
-    });
-}
-
-function getBackupUrl(app, callback) {
-    assert(!app || typeof app === 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var filename = '';
-    if (app) {
-        filename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-    } else {
-        filename = util.format('backup_%s-v%s.tar.gz', (new Date()).toISOString(), config.version());
-    }
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
-            if (error) return callback(error);
-
-            var obj = {
-                id: filename,
-                url: result.url,
-                sessionToken: result.sessionToken,
-                backupKey: backupConfig.key
-            };
-
-            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
-
-            callback(null, obj);
-        });
-    });
-}
-
-// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
-function getRestoreUrl(backupId, callback) {
-    assert.strictEqual(typeof backupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
-            if (error) return callback(error);
-
-            var obj = {
-                id: backupId,
-                url: result.url,
-                sessionToken: result.sessionToken,
-                backupKey: backupConfig.key
-            };
-
-            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
-
-            callback(null, obj);
-        });
-    });
-}
-
-function copyLastBackup(app, callback) {
-    assert(app && typeof app === 'object');
-    assert.strictEqual(typeof app.lastBackupId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var toFilename = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
-
-    settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
-
-        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilename, function (error) {
-            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-
-            return callback(null, toFilename);
-        });
-    });
-}

src/cert/acme.js

@@ -1,425 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-var assert = require('assert'),
-    async = require('async'),
-    crypto = require('crypto'),
-    debug = require('debug')('box:cert/acme'),
-    fs = require('fs'),
-    path = require('path'),
-    paths = require('../paths.js'),
-    safe = require('safetydance'),
-    superagent = require('superagent'),
-    ursa = require('ursa'),
-    util = require('util'),
-    _ = require('underscore');
-
-var CA_PROD = 'https://acme-v01.api.letsencrypt.org',
-    CA_STAGING = 'https://acme-staging.api.letsencrypt.org',
-    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
-
-exports = module.exports = {
-    getCertificate: getCertificate
-};
-
-function AcmeError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
-        this.message = reason;
-    } else if (typeof errorOrMessage === 'string') {
-        this.message = errorOrMessage;
-    } else {
-        this.message = 'Internal error';
-        this.nestedError = errorOrMessage;
-    }
-}
-util.inherits(AcmeError, Error);
-AcmeError.INTERNAL_ERROR = 'Internal Error';
-AcmeError.EXTERNAL_ERROR = 'External Error';
-AcmeError.ALREADY_EXISTS = 'Already Exists';
-AcmeError.NOT_COMPLETED = 'Not Completed';
-AcmeError.FORBIDDEN = 'Forbidden';
-
-// http://jose.readthedocs.org/en/latest/
-// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
-// https://community.letsencrypt.org/t/list-of-client-implementations/2103
-
-function Acme(options) {
-    assert.strictEqual(typeof options, 'object');
-
-    this.caOrigin = options.prod ? CA_PROD : CA_STAGING;
-    this.accountKeyPem = null; // Buffer
-
-    this.chainPem = options.prod ? safe.fs.readFileSync(__dirname + '/lets-encrypt-x1-cross-signed.pem.txt') : new Buffer('');
-}
-
-Acme.prototype.getNonce = function (callback) {
-    superagent.get(this.caOrigin + '/directory', function (error, response) {
-        if (error) return callback(error);
-        if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
-
-        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
-    });
-};
-
-// urlsafe base64 encoding (jose)
-function urlBase64Encode(string) {
-    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
-}
-
-function b64(str) {
-    var buf = util.isBuffer(str) ? str : new Buffer(str);
-   return urlBase64Encode(buf.toString('base64'));
-}
-
-Acme.prototype.sendSignedRequest = function (url, payload, callback) {
-    assert.strictEqual(typeof url, 'string');
-    assert.strictEqual(typeof payload, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    assert(util.isBuffer(this.accountKeyPem));
-    var privateKey = ursa.createPrivateKey(this.accountKeyPem);
-
-    var header = {
-        alg: 'RS256',
-        jwk: {
-            e: b64(privateKey.getExponent()),
-            kty: 'RSA',
-            n: b64(privateKey.getModulus())
-        }
-    };
- 
-    var payload64 = b64(payload);
-
-    this.getNonce(function (error, nonce) {
-        if (error) return callback(error);
-
-        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
-
-        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
-
-        var signer = ursa.createSigner('sha256');
-        signer.update(protected64 + '.' + payload64, 'utf8');
-        var signature64 = urlBase64Encode(signer.sign(privateKey, 'base64'));
-
-        var data = {
-            header: header,
-            protected: protected64,
-            payload: payload64,
-            signature: signature64
-        };
-
-        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).end(function (error, res) {
-            if (error && !error.response) return callback(error); // network errors
-
-            callback(null, res);
-        });
-    });
-};
-
-Acme.prototype.registerUser = function (email, callback) {
-    assert.strictEqual(typeof email, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var payload = {
-        resource: 'new-reg',
-        contact: [ 'mailto:' + email ],
-        agreement: LE_AGREEMENT
-    };
-
-    debug('registerUser: %s', email);
-
-    this.sendSignedRequest(this.caOrigin + '/acme/new-reg', JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
-        if (result.statusCode === 409) return callback(new AcmeError(AcmeError.ALREADY_EXISTS, result.body.detail));
-        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
-
-        debug('registerUser: registered user %s', email);
-
-        callback();
-    });
-};
-
-Acme.prototype.registerDomain = function (domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var payload = {
-        resource: 'new-authz',
-        identifier: {
-            type: 'dns',
-            value: domain
-        }
-    };
-
-    debug('registerDomain: %s', domain);
-
-    this.sendSignedRequest(this.caOrigin + '/acme/new-authz', JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering domain: ' + error.message));
-        if (result.statusCode === 403) return callback(new AcmeError(AcmeError.FORBIDDEN, result.body.detail));
-        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
-
-        debug('registerDomain: registered %s', domain);
-
-        callback(null, result.body);
-    });
-};
-
-Acme.prototype.prepareHttpChallenge = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
-
-    var token = challenge.token;
-
-    assert(util.isBuffer(this.accountKeyPem));
-    var privateKey = ursa.createPrivateKey(this.accountKeyPem);
-
-    var jwk = {
-        e: b64(privateKey.getExponent()),
-        kty: 'RSA',
-        n: b64(privateKey.getModulus())
-    };
-
-    var shasum = crypto.createHash('sha256');
-    shasum.update(JSON.stringify(jwk));
-    var thumbprint = urlBase64Encode(shasum.digest('base64'));
-    var keyAuthorization = token + '.' + thumbprint;
-
-    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, token));
-
-    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, token), token + '.' + thumbprint, function (error) {
-        if (error) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, error));
-
-        callback();
-    });
-};
-
-Acme.prototype.notifyChallengeReady = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('notifyChallengeReady: %s was met', challenge.uri);
-
-    var keyAuthorization = fs.readFileSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), 'utf8');
-
-    var payload = {
-        resource: 'challenge',
-        keyAuthorization: keyAuthorization
-    };
-
-    this.sendSignedRequest(challenge.uri, JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when notifying challenge: ' + error.message));
-        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 202, got %s %s', result.statusCode, result.text)));
-
-        callback();
-    });
-};
-
-Acme.prototype.waitForChallenge = function (challenge, callback) {
-    assert.strictEqual(typeof challenge, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('waitingForChallenge: %j', challenge);
-
-    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
-        debug('waitingForChallenge: getting status');
-
-        superagent.get(challenge.uri).end(function (error, result) {
-            if (error && !error.response) {
-                debug('waitForChallenge: network error getting uri %s', challenge.uri);
-                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, error.message)); // network error
-            }
-            if (result.statusCode !== 202) {
-                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
-                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
-            }
-
-            debug('waitForChallenge: status is "%s"', result.body.status);
-
-            if (result.body.status === 'pending') return retryCallback(new AcmeError(AcmeError.NOT_COMPLETED));
-            else if (result.body.status === 'valid') return retryCallback();
-            else return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
-        });
-    }, function retryFinished(error) {
-        // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
-        callback(error);
-    });
-};
-
-// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
-Acme.prototype.signCertificate = function (domain, csrDer, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert(util.isBuffer(csrDer));
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-
-    var payload = {
-        resource: 'new-cert',
-        csr: b64(csrDer)
-    };
-
-    debug('signCertificate: sending new-cert request');
-
-    this.sendSignedRequest(this.caOrigin + '/acme/new-cert', JSON.stringify(payload), function (error, result) {
-        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when signing certificate: ' + error.message));
-        // 429 means we reached the cert limit for this domain
-        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 201, got %s %s', result.statusCode, result.text)));
-
-        var certUrl = result.headers.location;
-
-        if (!certUrl) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Missing location in downloadCertificate'));
-
-        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // for renewal
-
-        return callback(null, result.headers.location);
-    });
-};
-
-Acme.prototype.createKeyAndCsr = function (domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    var execSync = safe.child_process.execSync;
-
-    var privateKeyFile = path.join(outdir, domain + '.key');
-    var key = execSync('openssl genrsa 4096');
-    if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-
-    debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
-
-    var csrDer = execSync(util.format('openssl req -new -key %s -outform DER -subj /CN=%s', privateKeyFile, domain));
-    if (!csrDer) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-    var csrFile = path.join(outdir, domain + '.csr');
-    if (!safe.fs.writeFileSync(csrFile, csrFile)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-
-    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
-
-    callback(null, csrDer);
-};
-
-Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof certUrl, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    var that = this;
-
-    superagent.get(certUrl).buffer().parse(function (res, done) {
-        var data = [ ];
-        res.on('data', function(chunk) { data.push(chunk); });
-        res.on('end', function () { res.text = Buffer.concat(data); done(); });
-    }).end(function (error, result) {
-        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
-        if (result.statusCode === 202) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, 'Retry not implemented yet'));
-        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
-
-        var certificateDer = result.text;
-        var execSync = safe.child_process.execSync;
-
-        safe.fs.writeFileSync(path.join(outdir, domain + '.der'), certificateDer);
-        debug('downloadCertificate: cert der file saved');
-
-        var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { input: certificateDer }); // this is really just base64 encoding with header
-        if (!certificatePem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-
-        var certificateFile = path.join(outdir, domain + '.cert');
-        var fullChainPem = Buffer.concat([certificatePem, that.chainPem]);
-        if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-
-        debug('downloadCertificate: cert file saved at %s', certificateFile);
-
-        callback();
-    });
-};
-
-Acme.prototype.acmeFlow = function (domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
-    // we cannot use admin@fqdn because the user might not have set it up.
-    // we cannot use owner email because we don't have it yet (the admin cert is fetched before activation)
-    // one option is to update the owner email when a second cert is requested (https://github.com/ietf-wg-acme/acme/issues/30)
-    var email = 'admin@cloudron.io';
-
-    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
-        debug('getCertificate: generating acme account key on first run');
-        this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
-        if (!this.accountKeyPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
-
-        safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
-    } else {
-        debug('getCertificate: using existing acme account key');
-        this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
-    }
-
-    var that = this;
-    that.registerUser(email, function (error) {
-        if (error && error.reason !== AcmeError.ALREADY_EXISTS) return callback(error);
-
-        that.registerDomain(domain, function (error, result) {
-            if (error) return callback(error);
-
-            debug('acmeFlow: challenges: %j', result);
-
-            var httpChallenges = result.challenges.filter(function(x) { return x.type === 'http-01'; });
-            if (httpChallenges.length === 0) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'no http challenges'));
-            var challenge = httpChallenges[0];
-
-            async.waterfall([
-                that.prepareHttpChallenge.bind(that, challenge),
-                that.notifyChallengeReady.bind(that, challenge),
-                that.waitForChallenge.bind(that, challenge),
-                that.createKeyAndCsr.bind(that, domain),
-                that.signCertificate.bind(that, domain),
-                that.downloadCertificate.bind(that, domain)
-            ], callback);
-        });
-    });
-};
-
-Acme.prototype.getCertificate = function (domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var outdir = paths.APP_CERTS_DIR;
-    var certUrl = safe.fs.readFileSync(path.join(outdir, domain + '.url'), 'utf8');
-    var certificateGetter;
-
-    if (certUrl) {
-        debug('getCertificate: renewing existing cert for %s from %s', domain, certUrl);
-        certificateGetter = this.downloadCertificate.bind(this, domain, certUrl);
-    } else {
-        debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
-        certificateGetter = this.acmeFlow.bind(this, domain);
-    }
-
-    certificateGetter(function (error) {
-        if (error) return callback(error);
-
-        callback(null, path.join(outdir, domain + '.cert'), path.join(outdir, domain + '.key'));
-    });
-};
-
-function getCertificate(domain, options, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var acme = new Acme(options || { });
-    acme.getCertificate(domain, callback);
-}

src/cert/caas.js

@@ -1,18 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    getCertificate: getCertificate
-};
-
-var assert = require('assert'),
-	debug = require('debug')('box:cert/caas.js');
-
-function getCertificate(domain, options, callback) {
-	assert.strictEqual(typeof domain, 'string');
-	assert.strictEqual(typeof options, 'object');
-	assert.strictEqual(typeof callback, 'function');
-
-    debug('getCertificate: using fallback certificate', domain);
-
-    return callback(null, 'cert/host.cert', 'cert/host.key');
-}

src/cert/lets-encrypt-x1-cross-signed.pem.txt

@@ -1,27 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIEqDCCA5CgAwIBAgIRAJgT9HUT5XULQ+dDHpceRL0wDQYJKoZIhvcNAQELBQAw
-PzEkMCIGA1UEChMbRGlnaXRhbCBTaWduYXR1cmUgVHJ1c3QgQ28uMRcwFQYDVQQD
-Ew5EU1QgUm9vdCBDQSBYMzAeFw0xNTEwMTkyMjMzMzZaFw0yMDEwMTkyMjMzMzZa
-MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
-ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMTCCASIwDQYJKoZIhvcNAQEBBQAD
-ggEPADCCAQoCggEBAJzTDPBa5S5Ht3JdN4OzaGMw6tc1Jhkl4b2+NfFwki+3uEtB
-BaupnjUIWOyxKsRohwuj43Xk5vOnYnG6eYFgH9eRmp/z0HhncchpDpWRz/7mmelg
-PEjMfspNdxIknUcbWuu57B43ABycrHunBerOSuu9QeU2mLnL/W08lmjfIypCkAyG
-dGfIf6WauFJhFBM/ZemCh8vb+g5W9oaJ84U/l4avsNwa72sNlRZ9xCugZbKZBDZ1
-gGusSvMbkEl4L6KWTyogJSkExnTA0DHNjzE4lRa6qDO4Q/GxH8Mwf6J5MRM9LTb4
-4/zyM2q5OTHFr8SNDR1kFjOq+oQpttQLwNh9w5MCAwEAAaOCAZIwggGOMBIGA1Ud
-EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMH8GCCsGAQUFBwEBBHMwcTAy
-BggrBgEFBQcwAYYmaHR0cDovL2lzcmcudHJ1c3RpZC5vY3NwLmlkZW50cnVzdC5j
-b20wOwYIKwYBBQUHMAKGL2h0dHA6Ly9hcHBzLmlkZW50cnVzdC5jb20vcm9vdHMv
-ZHN0cm9vdGNheDMucDdjMB8GA1UdIwQYMBaAFMSnsaR7LHH62+FLkHX/xBVghYkQ
-MFQGA1UdIARNMEswCAYGZ4EMAQIBMD8GCysGAQQBgt8TAQEBMDAwLgYIKwYBBQUH
-AgEWImh0dHA6Ly9jcHMucm9vdC14MS5sZXRzZW5jcnlwdC5vcmcwPAYDVR0fBDUw
-MzAxoC+gLYYraHR0cDovL2NybC5pZGVudHJ1c3QuY29tL0RTVFJPT1RDQVgzQ1JM
-LmNybDATBgNVHR4EDDAKoQgwBoIELm1pbDAdBgNVHQ4EFgQUqEpqYwR93brm0Tm3
-pkVl7/Oo7KEwDQYJKoZIhvcNAQELBQADggEBANHIIkus7+MJiZZQsY14cCoBG1hd
-v0J20/FyWo5ppnfjL78S2k4s2GLRJ7iD9ZDKErndvbNFGcsW+9kKK/TnY21hp4Dd
-ITv8S9ZYQ7oaoqs7HwhEMY9sibED4aXw09xrJZTC9zK1uIfW6t5dHQjuOWv+HHoW
-ZnupyxpsEUlEaFb+/SCI4KCSBdAsYxAcsHYI5xxEI4LutHp6s3OT2FuO90WfdsIk
-6q78OMSdn875bNjdBYAqxUp2/LEIHfDBkLoQz0hFJmwAbYahqKaLn73PAAm1X2kj
-f1w8DdnkabOLGeOVcj9LQ+s67vBykx4anTjURkbqZslUEUsn2k5xeua2zUk=
------END CERTIFICATE-----

src/certificates.js

@@ -1,250 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-var acme = require('./cert/acme.js'),
-    assert = require('assert'),
-    async = require('async'),
-    caas = require('./cert/caas.js'),
-    cloudron = require('./cloudron.js'),
-    config = require('./config.js'),
-    constants = require('./constants.js'),
-    debug = require('debug')('box:src/certificates'),
-    fs = require('fs'),
-    nginx = require('./nginx.js'),
-    path = require('path'),
-    paths = require('./paths.js'),
-    safe = require('safetydance'),
-    settings = require('./settings.js'),
-    sysinfo = require('./sysinfo.js'),
-    util = require('util'),
-    waitForDns = require('./waitfordns.js'),
-    x509 = require('x509');
-
-exports = module.exports = {
-    installAdminCertificate: installAdminCertificate,
-    autoRenew: autoRenew,
-    setFallbackCertificate: setFallbackCertificate,
-    setAdminCertificate: setAdminCertificate,
-    CertificatesError: CertificatesError,
-    validateCertificate: validateCertificate,
-    ensureCertificate: ensureCertificate
-};
-
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
-function CertificatesError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
-        this.message = reason;
-    } else if (typeof errorOrMessage === 'string') {
-        this.message = errorOrMessage;
-    } else {
-        this.message = 'Internal error';
-        this.nestedError = errorOrMessage;
-    }
-}
-util.inherits(CertificatesError, Error);
-CertificatesError.INTERNAL_ERROR = 'Internal Error';
-CertificatesError.INVALID_CERT = 'Invalid certificate';
-
-function getApi(callback) {
-    settings.getTlsConfig(function (error, tlsConfig) {
-        if (error) return callback(error);
-
-        var api = tlsConfig.provider === 'caas' ? caas : acme;
-
-        var options = { };
-        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
-
-        callback(null, api, options);
-    });
-}
-
-function installAdminCertificate(callback) {
-    if (cloudron.isConfiguredSync()) return callback();
-
-    settings.getTlsConfig(function (error, tlsConfig) {
-        if (error) return callback(error);
-
-        if (tlsConfig.provider === 'caas') return callback();
-
-        sysinfo.getIp(function (error, ip) {
-            if (error) return callback(error);
-
-            waitForDns(config.adminFqdn(), ip, config.fqdn(), function (error) {
-                if (error) return callback(error); // this cannot happen because we retry forever
-
-                ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
-                    if (error) { // currently, this can never happen
-                        debug('Error obtaining certificate. Proceed anyway', error);
-                        return callback();
-                    }
-
-                    nginx.configureAdmin(certFilePath, keyFilePath, callback);
-                });
-            });
-        });
-    });
-}
-
-function needsRenewalSync(certFilePath) {
-    var result = safe.child_process.execSync('openssl x509 -checkend %s -in %s', 60 * 60 * 24 * 5, certFilePath);
-
-    return result === null; // command errored
-}
-
-function autoRenew(callback) {
-    debug('autoRenew: Checking certificates for renewal');
-    callback = callback || NOOP_CALLBACK;
-
-    var filenames = safe.fs.readdirSync(paths.APP_CERTS_DIR);
-    if (!filenames) {
-        debug('autoRenew: Error getting filenames: %s', safe.error.message);
-        return;
-    }
-
-    var certs = filenames.filter(function (f) {
-        return f.match(/\.cert$/) !== null && needsRenewalSync(path.join(paths.APP_CERTS_DIR, f));
-    });
-
-    debug('autoRenew: %j needs to be renewed', certs);
-
-    getApi(function (error, api, apiOptions) {
-        if (error) return callback(error);
-
-        async.eachSeries(certs, function iterator(cert, iteratorCallback) {
-            var domain = cert.match(/^(.*)\.cert$/)[1];
-            if (domain === 'host') return iteratorCallback(); // cannot renew fallback cert
-
-            debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
-
-            api.getCertificate(domain, apiOptions, function (error) {
-                if (error) debug('autoRenew: could not renew cert for %s', domain, error);
-
-                iteratorCallback(); // move on to next cert
-            });
-        });
-    });
-}
-
-// note: https://tools.ietf.org/html/rfc4346#section-7.4.2 (certificate_list) requires that the
-// servers certificate appears first (and not the intermediate cert)
-function validateCertificate(cert, key, fqdn) {
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof fqdn, 'string');
-
-    if (cert === null && key === null) return null;
-    if (!cert && key) return new Error('missing cert');
-    if (cert && !key) return new Error('missing key');
-
-    var content;
-    try {
-        content = x509.parseCert(cert);
-    } catch (e) {
-        return new Error('invalid cert: ' + e.message);
-    }
-
-    // check expiration
-    if (content.notAfter < new Date()) return new Error('cert expired');
-
-    function matchesDomain(domain) {
-        if (domain === fqdn) return true;
-        if (domain.indexOf('*') === 0 && domain.slice(2) === fqdn.slice(fqdn.indexOf('.') + 1)) return true;
-
-        return false;
-    }
-
-    // check domain
-    var domains = content.altNames.concat(content.subject.commonName);
-    if (!domains.some(matchesDomain)) return new Error(util.format('cert is not valid for this domain. Expecting %s in %j', fqdn, domains));
-
-    // http://httpd.apache.org/docs/2.0/ssl/ssl_faq.html#verify
-    var certModulus = safe.child_process.execSync('openssl x509 -noout -modulus', { encoding: 'utf8', input: cert });
-    var keyModulus = safe.child_process.execSync('openssl rsa -noout -modulus', { encoding: 'utf8', input: key });
-    if (certModulus !== keyModulus) return new Error('key does not match the cert');
-
-    return null;
-}
-
-function setFallbackCertificate(cert, key, callback) {
-    assert.strictEqual(typeof cert, 'string');
-    assert.strictEqual(typeof key, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = validateCertificate(cert, key, '*.' + config.fqdn());
-    if (error) return callback(new CertificatesError(CertificatesError.INVALID_CERT, error.message));
-
-    // backup the cert
-    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.cert'), cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
-    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.key'), key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
-
-    // copy over fallback cert
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
-
-    nginx.reload(function (error) {
-        if (error) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, error));
-
-        return callback(null);
-    });
-}
-
-function setAdminCertificate(cert, key, callback) {
-    assert.strictEqual(typeof cert, 'string');
-    assert.strictEqual(typeof key, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
-    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
-    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
-
-    var error = validateCertificate(cert, key, vhost);
-    if (error) return callback(new CertificatesError(CertificatesError.INVALID_CERT, error.message));
-
-    // backup the cert
-    if (!safe.fs.writeFileSync(certFilePath, cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
-    if (!safe.fs.writeFileSync(keyFilePath, key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
-
-    nginx.configureAdmin(certFilePath, keyFilePath, callback);
-}
-
-function ensureCertificate(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
-    var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
-    var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
-
-    if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
-        debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);
-
-        if (!needsRenewalSync(userCertFilePath)) return callback(null, userCertFilePath, userKeyFilePath);
-
-        debug('ensureCertificate: %s cert require renewal', domain);
-    }
-
-    getApi(function (error, api, apiOptions) {
-        if (error) return callback(error);
-
-        debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);
-
-        api.getCertificate(domain, apiOptions, function (error, certFilePath, keyFilePath) {
-            if (error) {
-                debug('ensureCertificate: could not get certificate. using fallback certs', error);
-                return callback(null, 'cert/host.cert', 'cert/host.key'); // use fallback certs
-            }
-
-            callback(null, certFilePath, keyFilePath);
-        });
-    });
-}

src/clientdb.js

@@ -2,152 +2,170 @@
 
 'use strict';
 
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:clientdb');
+
 exports = module.exports = {
     get: get,
     getAll: getAll,
-    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
+    getAllWithDetails: getAllWithDetails,
+    getByClientId: getByClientId,
     add: add,
     del: del,
+    replaceByAppId: replaceByAppId,
     getByAppId: getByAppId,
-    getByAppIdAndType: getByAppIdAndType,
-
     delByAppId: delByAppId,
-    delByAppIdAndType: delByAppIdAndType,
-
-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    clear: clear
 };
 
-var assert = require('assert'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js');
-
-var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
+var CLIENTS_FIELDS = [ 'id', 'appId', 'clientId', 'clientSecret', 'name', 'redirectURI', 'scope' ].join(',');
+var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.clientId', 'clients.clientSecret', 'clients.name', 'clients.redirectURI', 'clients.scope' ].join(',');
 
 function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
+    database.get('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, result[0]);
+        callback(null, result);
     });
 }
 
 function getAll(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients ORDER BY appId', function (error, results) {
+    database.all('SELECT ' + CLIENTS_FIELDS + ' FROM clients', [ ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (typeof results === 'undefined') results = [];
 
         callback(null, results);
     });
 }
 
-function getAllWithTokenCountByIdentifier(identifier, callback) {
-    assert.strictEqual(typeof identifier, 'string');
-    assert.strictEqual(typeof callback, 'function');
+function getAllWithDetails(callback) {
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
+    // TODO should this be per user?
+    database.all('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        callback(null, results);
+        if (typeof results === 'undefined') results = [];
 
+        callback(null, results);
+    });
+}
+
+function getByClientId(clientId, callback) {
+    assert(typeof clientId === 'string');
+    assert(typeof callback === 'function');
+
+    database.get('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE clientId = ? LIMIT 1', [ clientId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null, result);
     });
 }
 
 function getByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
+    database.get('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        return callback(null, result);
     });
 }
 
-function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
+function add(id, appId, clientId, clientSecret, name, redirectURI, scope, callback) {
+    assert(typeof id === 'string');
+    assert(typeof appId === 'string');
+    assert(typeof clientId === 'string');
+    assert(typeof clientSecret === 'string');
+    assert(typeof name === 'string');
+    assert(typeof redirectURI === 'string');
+    assert(typeof scope === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    var data = {
+        $id: id,
+        $appId: appId,
+        $clientId: clientId,
+        $clientSecret: clientSecret,
+        $name: name,
+        $redirectURI: redirectURI,
+        $scope: scope
+    };
 
-        return callback(null, result[0]);
-    });
-}
-
-function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
-
-    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.run('INSERT INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES ($id, $appId, $clientId, $clientSecret, $name, $redirectURI, $scope)', data, function (error) {
+        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
     });
 }
 
 function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof id === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM clients WHERE id = ?', [ id ], function (error, result) {
+    database.run('DELETE FROM clients WHERE id = ?', [ id ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         callback(null);
     });
 }
 
 function delByAppId(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof appId === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM clients WHERE appId=?', [ appId ], function (error, result) {
+    database.run('DELETE FROM clients WHERE appId=?', [ appId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         return callback(null);
     });
 }
 
-function delByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof callback, 'function');
+function replaceByAppId(id, appId, clientId, clientSecret, name, redirectURI, scope, callback) {
+    assert(typeof id === 'string');
+    assert(typeof appId === 'string');
+    assert(typeof clientId === 'string');
+    assert(typeof clientSecret === 'string');
+    assert(typeof name === 'string');
+    assert(typeof redirectURI === 'string');
+    assert(typeof scope === 'string');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    var data = {
+        $id: id,
+        $appId: appId,
+        $clientId: clientId,
+        $clientSecret: clientSecret,
+        $name: name,
+        $redirectURI: redirectURI,
+        $scope: scope
+    };
 
-        return callback(null);
+    database.run('INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES ($id, $appId, $clientId, $clientSecret, $name, $redirectURI, $scope)', data, function (error) {
+        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
     });
 }
 
 function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.run('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         return callback(null);

src/clients.js

@@ -1,186 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    ClientsError: ClientsError,
-
-    add: add,
-    get: get,
-    del: del,
-    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
-    getClientTokensByUserId: getClientTokensByUserId,
-    delClientTokensByUserId: delClientTokensByUserId
-};
-
-var assert = require('assert'),
-    util = require('util'),
-    hat = require('hat'),
-    appdb = require('./appdb.js'),
-    tokendb = require('./tokendb.js'),
-    constants = require('./constants.js'),
-    async = require('async'),
-    clientdb = require('./clientdb.js'),
-    DatabaseError = require('./databaseerror.js'),
-    uuid = require('node-uuid');
-
-function ClientsError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
-        this.message = reason;
-    } else if (typeof errorOrMessage === 'string') {
-        this.message = errorOrMessage;
-    } else {
-        this.message = 'Internal error';
-        this.nestedError = errorOrMessage;
-    }
-}
-util.inherits(ClientsError, Error);
-ClientsError.INVALID_SCOPE = 'Invalid scope';
-ClientsError.INVALID_CLIENT = 'Invalid client';
-
-function validateScope(scope) {
-    assert.strictEqual(typeof scope, 'string');
-
-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
-
-    // TODO maybe validate all individual scopes if they exist
-
-    return null;
-}
-
-function add(appId, type, redirectURI, scope, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = validateScope(scope);
-    if (error) return callback(error);
-
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
-
-    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
-        if (error) return callback(error);
-
-        var client = {
-            id: id,
-            appId: appId,
-            type: type,
-            clientSecret: clientSecret,
-            redirectURI: redirectURI,
-            scope: scope
-        };
-
-        callback(null, client);
-    });
-}
-
-function get(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.get(id, function (error, result) {
-        if (error) return callback(error);
-        callback(null, result);
-    });
-}
-
-function del(id, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.del(id, function (error, result) {
-        if (error) return callback(error);
-        callback(null, result);
-    });
-}
-
-function getAllWithDetailsByUserId(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
-        if (error) return callback(error);
-
-        var tmp = [];
-        async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
-                record.name = constants.ADMIN_NAME;
-                record.location = constants.ADMIN_LOCATION;
-
-                tmp.push(record);
-
-                return callback(null);
-            }
-
-            appdb.get(record.appId, function (error, result) {
-                if (error) {
-                    console.error('Failed to get app details for oauth client', result, error);
-                    return callback(null);  // ignore error so we continue listing clients
-                }
-
-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
-
-                record.location = result.location;
-
-                tmp.push(record);
-
-                callback(null);
-            });
-        }, function (error) {
-            if (error) return callback(error);
-            callback(null, tmp);
-        });
-    });
-}
-
-function getClientTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null, []);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null, result || []);
-    });
-}
-
-function delClientTokensByUserId(clientId, userId, callback) {
-    assert.strictEqual(typeof clientId, 'string');
-    assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) {
-            // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
-                if (error) return callback(error);
-                callback(null);
-            });
-            return;
-        }
-        if (error) return callback(error);
-        callback(null);
-    });
-}

src/cloudron.js

@@ -2,6 +2,7 @@
 
 'use strict';
 
+// intentionally placed here because of circular dep with updater
 exports = module.exports = {
     CloudronError: CloudronError,
 
@@ -10,81 +11,46 @@ exports = module.exports = {
     activate: activate,
     getConfig: getConfig,
     getStatus: getStatus,
-
-    sendHeartbeat: sendHeartbeat,
-
-    update: update,
-    reboot: reboot,
-    migrate: migrate,
     backup: backup,
-    ensureBackup: ensureBackup,
 
-    isConfiguredSync: isConfiguredSync,
+    getBackupUrl: getBackupUrl,
+    setCertificate: setCertificate,
 
-    events: new (require('events').EventEmitter)(),
-
-    EVENT_ACTIVATED: 'activated',
-    EVENT_CONFIGURED: 'configured'
+    getIp: getIp
 };
 
-var apps = require('./apps.js'),
-    AppsError = require('./apps.js').AppsError,
-    assert = require('assert'),
-    async = require('async'),
-    backups = require('./backups.js'),
-    BackupsError = require('./backups.js').BackupsError,
-    clientdb = require('./clientdb.js'),
-    config = require('./config.js'),
+var assert = require('assert'),
+    config = require('../config.js'),
     debug = require('debug')('box:cloudron'),
+    clientdb = require('./clientdb.js'),
+    execFile = require('child_process').execFile,
     fs = require('fs'),
-    locker = require('./locker.js'),
     os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
-    progress = require('./progress.js'),
     safe = require('safetydance'),
-    settings = require('./settings.js'),
-    shell = require('./shell.js'),
-    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
-    sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
-    updateChecker = require('./updatechecker.js'),
+    updater = require('./updater.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
     util = require('util'),
-    webhooks = require('./webhooks.js');
+    uuid = require('node-uuid'),
+    _ = require('underscore');
 
-var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
-    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
-    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
+var SUDO = '/usr/bin/sudo',
+    TAR = os.platform() === 'darwin' ? '/usr/bin/tar' : '/bin/tar',
+    BACKUP_CMD = path.join(__dirname, 'scripts/backup.sh'),
+    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
 
-var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
-var gUpdatingDns = false,                // flag for dns update reentrancy
-    gCloudronDetails = null,             // cached cloudron details like region,size...
-    gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
-
-function debugApp(app, args) {
-    assert(!app || typeof app === 'object');
-
-    var prefix = app ? app.location : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
-function ignoreError(func) {
-    return function (callback) {
-        func(function (error) {
-            if (error) console.error('Ignored error:', error);
-            callback();
-        });
-    };
-}
+var gBackupTimerId = null,
+    gAddMailDnsRecordsTimerId = null,
+    gGetCertificateTimerId = null,
+    gCachedIp = null;
 
 function CloudronError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
+    assert(typeof reason === 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
 
     Error.call(this);
@@ -104,633 +70,249 @@ function CloudronError(reason, errorOrMessage) {
 util.inherits(CloudronError, Error);
 CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
-CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
-CloudronError.BAD_STATE = 'Bad state';
-CloudronError.NOT_FOUND = 'Not found';
+CloudronError.APPSTORE_DOWN = 'Appstore Down';
 
 function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
+    // every backup restarts the box. the setInterval is only needed should that fail for some reason
+    gBackupTimerId = setInterval(backup, 4 * 60 * 60 * 1000);
 
-    syncConfigState(callback);
-}
+    sendHeartBeat();
 
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
+    if (process.env.NODE_ENV !== 'test') {
+        addMailDnsRecords();
+    }
 
     callback(null);
 }
 
-function isConfiguredSync() {
-    return gIsConfigured === true;
+function uninitialize(callback) {
+    assert(typeof callback === 'function');
+
+    clearInterval(gBackupTimerId);
+    gBackupTimerId = null;
+
+    clearTimeout(gAddMailDnsRecordsTimerId);
+    gAddMailDnsRecordsTimerId = null;
+
+    clearTimeout(gGetCertificateTimerId);
+    gGetCertificateTimerId = null;
+
+    gCachedIp = null;
+
+    callback(null);
 }
 
-function isConfigured(callback) {
-    // set of rules to see if we have the configs required for cloudron to function
-    // note this checks for missing configs and not invalid configs
-
-    settings.getDnsConfig(function (error, dnsConfig) {
-        if (error) return callback(error);
-
-        if (!dnsConfig) return callback(null, false);
-
-        var isConfigured = (config.isCustomDomain() && dnsConfig.provider === 'route53') ||
-                        (!config.isCustomDomain() && dnsConfig.provider === 'caas');
-
-        callback(null, isConfigured);
-    });
-}
-
-function syncConfigState(callback) {
-    assert(!gIsConfigured);
-
-    callback = callback || NOOP_CALLBACK;
-
-    isConfigured(function (error, configured) {
-        if (error) return callback(error);
-
-        debug('syncConfigState: configured = %s', configured);
-
-        if (configured) {
-            exports.events.emit(exports.EVENT_CONFIGURED);
-        } else {
-            settings.events.once(settings.DNS_CONFIG_KEY, function () { syncConfigState(); }); // check again later
-        }
-
-        gIsConfigured = configured;
-
-        callback();
-    });
-}
-
-function setTimeZone(ip, callback) {
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('setTimeZone ip:%s', ip);
-
-    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
-        if ((error && !error.response) || result.statusCode !== 200) {
-            debug('Failed to get geo location: %s', error.message);
-            return callback(null);
-        }
-
-        if (!result.body.timezone) {
-            debug('No timezone in geoip response : %j', result.body);
-            return callback(null);
-        }
-
-        debug('Setting timezone to ', result.body.timezone);
-
-        settings.setTimeZone(result.body.timezone, callback);
-    });
-}
-
-function activate(username, password, email, ip, callback) {
-    assert.strictEqual(typeof username, 'string');
-    assert.strictEqual(typeof password, 'string');
-    assert.strictEqual(typeof email, 'string');
-    assert.strictEqual(typeof ip, 'string');
-    assert.strictEqual(typeof callback, 'function');
+function activate(username, password, email, callback) {
+    assert(typeof username === 'string');
+    assert(typeof password === 'string');
+    assert(typeof email === 'string');
+    assert(typeof callback === 'function');
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
-
-    user.createOwner(username, password, email, function (error, userObject) {
+    user.create(username, password, email, true /* admin */, function (error) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error instanceof UserError) return callback(error);
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clientdb.getByAppId('webadmin', function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             // Also generate a token so the admin creation can also act as a login
             var token = tokendb.generateToken();
-            var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+            var expires = new Date(Date.now() + 60 * 60000).toUTCString(); // 1 hour
 
-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, username, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-                // EE API is sync. do not keep the REST API reponse waiting
-                process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });
-
                 callback(null, { token: token, expires: expires });
             });
         });
     });
 }
 
-function getStatus(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function getBackupUrl(callback) {
+    assert(typeof callback === 'function');
 
-    userdb.count(function (error, count) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+    if (!config.appServerUrl()) return callback(new Error('No appstore server url set'));
+    if (!config.token()) return callback(new Error('No appstore server token set'));
 
-        settings.getCloudronName(function (error, cloudronName) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+    var url = config.appServerUrl() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
 
-            callback(null, {
-                activated: count !== 0,
-                version: config.version(),
-                boxVersionsUrl: config.get('boxVersionsUrl'),
-                apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool
-                cloudronName: cloudronName
-            });
-        });
-    });
-}
+    superagent.put(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
+        if (error) return callback(new Error('Error getting presigned backup url: ' + error.message));
 
-function getCloudronDetails(callback) {
-    assert.strictEqual(typeof callback, 'function');
+        if (result.statusCode !== 201 || !result.body || !result.body.url) return callback(new Error('Error getting presigned backup url : ' + result.statusCode));
 
-    if (gCloudronDetails) return callback(null, gCloudronDetails);
-
-    if (!config.token()) {
-        gCloudronDetails = {
-            region: null,
-            size: null
-        };
-
-        return callback(null, gCloudronDetails);
-    }
-
-    superagent
-        .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
-        .query({ token: config.token() })
-        .end(function (error, result) {
-            if (error && !error.response) return callback(error);
-            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
-
-            gCloudronDetails = result.body.box;
-
-            return callback(null, gCloudronDetails);
-        });
-}
-
-function getConfig(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    getCloudronDetails(function (error, result) {
-        if (error) {
-            debug('Failed to fetch cloudron details.', error);
-
-            // set fallback values to avoid dependency on appstore
-            result = {
-                region: result ? result.region : null,
-                size: result ? result.size : null
-            };
-        }
-
-        settings.getCloudronName(function (error, cloudronName) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            settings.getDeveloperMode(function (error, developerMode) {
-                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-                sysinfo.getIp(function (error, ip) {
-                    if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-                    callback(null, {
-                        apiServerOrigin: config.apiServerOrigin(),
-                        webServerOrigin: config.webServerOrigin(),
-                        isDev: config.isDev(),
-                        fqdn: config.fqdn(),
-                        ip: ip,
-                        version: config.version(),
-                        update: updateChecker.getUpdateInfo(),
-                        progress: progress.get(),
-                        isCustomDomain: config.isCustomDomain(),
-                        developerMode: developerMode,
-                        region: result.region,
-                        size: result.size,
-                        memory: os.totalmem(),
-                        provider: config.provider(),
-                        cloudronName: cloudronName
-                    });
-                });
-            });
-        });
-    });
-}
-
-function sendHeartbeat() {
-    if (!config.token()) return;
-
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
-    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
-        if (error && !error.response) debug('Network error sending heartbeat.', error);
-        else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
-        else debug('Heartbeat sent to %s', url);
-    });
-}
-
-function readDkimPublicKeySync() {
-    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
-    var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
-
-    if (publicKey === null) {
-        debug('Error reading dkim public key.', safe.error);
-        return null;
-    }
-
-    // remove header, footer and new lines
-    publicKey = publicKey.split('\n').slice(1, -2).join('');
-
-    return publicKey;
-}
-
-function txtRecordsWithSpf(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    subdomains.get('', 'TXT', function (error, txtRecords) {
-        if (error) return callback(error);
-
-        debug('txtRecordsWithSpf: current txt records - %j', txtRecords);
-
-        var i, validSpf;
-
-        for (i = 0; i < txtRecords.length; i++) {
-            if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF
-
-            validSpf = txtRecords[i].indexOf(' a:' + config.fqdn() + ' ') !== -1;
-            break;
-        }
-
-        if (validSpf) return callback(null, null);
-
-        if (i == txtRecords.length) {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ~all"';
-        } else {
-            txtRecords[i] = '"v=spf1 a:' + config.fqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
-        }
-
-        return callback(null, txtRecords);
-    });
-}
-
-function addDnsRecords() {
-    var callback = NOOP_CALLBACK;
-
-    if (process.env.BOX_ENV === 'test') return callback();
-
-    if (gUpdatingDns) {
-        debug('addDnsRecords: dns update already in progress');
-        return callback();
-    }
-    gUpdatingDns = true;
-
-    var DKIM_SELECTOR = 'cloudron';
-    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
-
-    var dkimKey = readDkimPublicKeySync();
-    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
-
-    sysinfo.getIp(function (error, ip) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
-        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
-        // t=s limits the domainkey to this domain and not it's subdomains
-        var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
-        // DMARC requires special setup if report email id is in different domain
-        var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
-
-        var records = [ ];
-        if (config.isCustomDomain()) {
-            records.push(webadminRecord);
-            records.push(dkimRecord);
-        } else {
-            records.push(nakedDomainRecord);
-            records.push(webadminRecord);
-            records.push(dkimRecord);
-            records.push(dmarcRecord);
-        }
-
-        debug('addDnsRecords: %j', records);
-
-        async.retry({ times: 10, interval: 20000 }, function (retryCallback) {
-            txtRecordsWithSpf(function (error, txtRecords) {
-                if (error) return retryCallback(error);
-
-                if (txtRecords) records.push({ subdomain: '', type: 'TXT', values: txtRecords });
-
-                debug('addDnsRecords: will update %j', records);
-
-                async.mapSeries(records, function (record, iteratorCallback) {
-                    subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
-                }, function (error, changeIds) {
-                    if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
-                    else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
-
-                    retryCallback(error);
-                });
-            });
-        }, function (error) {
-            gUpdatingDns = false;
-
-            debug('addDnsRecords: done updating records with error:', error);
-
-            callback(error);
-        });
-    });
-}
-
-function reboot(callback) {
-    shell.sudo('reboot', [ REBOOT_CMD ], callback);
-}
-
-function migrate(size, region, callback) {
-    assert.strictEqual(typeof size, 'string');
-    assert.strictEqual(typeof region, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_MIGRATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    function unlock(error) {
-        if (error) {
-            debug('Failed to migrate', error);
-            locker.unlock(locker.OP_MIGRATE);
-        } else {
-            debug('Migration initiated successfully');
-            // do not unlock; cloudron is migrating
-        }
-
-        return;
-    }
-
-    // initiate the migration in the background
-    backupBoxAndApps(function (error, restoreKey) {
-        if (error) return unlock(error);
-
-        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
-
-        superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send({ size: size, region: region, restoreKey: restoreKey })
-          .end(function (error, result) {
-            if (error && !error.response) return unlock(error);
-            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
-
-            return unlock(null);
-        });
-    });
-
-    callback(null);
-}
-
-function update(boxUpdateInfo, callback) {
-    assert.strictEqual(typeof boxUpdateInfo, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!boxUpdateInfo) return callback(null);
-
-    var error = locker.lock(locker.OP_BOX_UPDATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    // initiate the update/upgrade but do not wait for it
-    if (boxUpdateInfo.upgrade) {
-        debug('Starting upgrade');
-        doUpgrade(boxUpdateInfo, function (error) {
-            if (error) {
-                console.error('Upgrade failed with error:', error);
-                locker.unlock(locker.OP_BOX_UPDATE);
-            }
-        });
-    } else {
-        debug('Starting update');
-        doUpdate(boxUpdateInfo, function (error) {
-            if (error) {
-                console.error('Update failed with error:', error);
-                locker.unlock(locker.OP_BOX_UPDATE);
-            }
-        });
-    }
-
-    callback(null);
-}
-
-function doUpgrade(boxUpdateInfo, callback) {
-    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
-
-    function upgradeError(e) {
-        progress.set(progress.UPDATE, -1, e.message);
-        callback(e);
-    }
-
-    progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
-
-    backupBoxAndApps(function (error) {
-        if (error) return upgradeError(error);
-
-        superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
-          .query({ token: config.token() })
-          .send({ version: boxUpdateInfo.version })
-          .end(function (error, result) {
-            if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
-            if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
-
-            progress.set(progress.UPDATE, 10, 'Updating base system');
-
-            // no need to unlock since this is the last thing we ever do on this box
-
-            callback(null);
-        });
-    });
-}
-
-function doUpdate(boxUpdateInfo, callback) {
-    assert(boxUpdateInfo && typeof boxUpdateInfo === 'object');
-
-    function updateError(e) {
-        progress.set(progress.UPDATE, -1, e.message);
-        callback(e);
-    }
-
-    progress.set(progress.UPDATE, 5, 'Backing up for update');
-
-    backupBoxAndApps(function (error) {
-        if (error) return updateError(error);
-
-        // fetch a signed sourceTarballUrl
-        superagent.get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/sourcetarballurl')
-          .query({ token: config.token(), boxVersion: boxUpdateInfo.version })
-          .end(function (error, result) {
-            if (error && !error.response) return updateError(new Error('Network error fetching sourceTarballUrl: ' + error));
-            if (result.statusCode !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.statusCode));
-            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + JSON.stringify(result.body)));
-
-            // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
-            var args = {
-                sourceTarballUrl: result.body.url,
-
-                // this data is opaque to the installer
-                data: {
-                    token: config.token(),
-                    apiServerOrigin: config.apiServerOrigin(),
-                    webServerOrigin: config.webServerOrigin(),
-                    fqdn: config.fqdn(),
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    isCustomDomain: config.isCustomDomain(),
-
-                    appstore: {
-                        token: config.token(),
-                        apiServerOrigin: config.apiServerOrigin()
-                    },
-                    caas: {
-                        token: config.token(),
-                        apiServerOrigin: config.apiServerOrigin(),
-                        webServerOrigin: config.webServerOrigin()
-                    },
-
-                    version: boxUpdateInfo.version,
-                    boxVersionsUrl: config.get('boxVersionsUrl')
-                }
-            };
-
-            debug('updating box %j', args);
-
-            superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
-                if (error && !error.response) return updateError(error);
-                if (result.statusCode !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));
-
-                progress.set(progress.UPDATE, 10, 'Updating cloudron software');
-
-                callback(null);
-            });
-        });
-
-        // Do not add any code here. The installer script will stop the box code any instant
+        return callback(null, result.body.url);
     });
 }
 
 function backup(callback) {
-    assert.strictEqual(typeof callback, 'function');
+    assert(typeof callback === 'function');
 
-    var error = locker.lock(locker.OP_FULL_BACKUP);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+    getBackupUrl(function (error, url) {
+        if (error) return callback(new CloudronError(CloudronError.APPSTORE_DOWN, error.message));
 
-    // clearing backup ensures tools can 'wait' on progress
-    progress.clear(progress.BACKUP);
+        debug('backup: url %s', url);
 
-    // start the backup operation in the background
-    backupBoxAndApps(function (error) {
-        if (error) console.error('backup failed.', error);
-
-        locker.unlock(locker.OP_FULL_BACKUP);
-    });
-
-    callback(null);
-}
-
-function ensureBackup(callback) {
-    callback = callback || function () { };
-
-    backups.getAllPaged(1, 1, function (error, backups) {
-        if (error) {
-            debug('Unable to list backups', error);
-            return callback(error); // no point trying to backup if appstore is down
-        }
-
-        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
-            debug('Previous backup was %j, no need to backup now', backups[0]);
-            return callback(null);
-        }
-
-        backup(callback);
-    });
-}
-
-function backupBoxWithAppBackupIds(appBackupIds, callback) {
-    assert(util.isArray(appBackupIds));
-
-    backups.getBackupUrl(null /* app */, function (error, result) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        debug('backup: url %s', result.url);
-
-        async.series([
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
+        execFile(SUDO, [ BACKUP_CMD,  url ], { }, function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-            debug('backup: successful');
-
-            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
-                if (error) return callback(error);
-                callback(null, result.id);
-            });
+            return callback(null);
         });
     });
 }
 
-// this function expects you to have a lock
-function backupBox(callback) {
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+function getIp() {
+    if (gCachedIp) return gCachedIp;
 
-        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
-        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+    var ifaces = os.networkInterfaces();
+    for (var dev in ifaces) {
+        if (dev.match(/^(en|eth|wlp).*/) === null) continue;
 
-        backupBoxWithAppBackupIds(appBackupIds, callback);
-    });
-}
-
-// this function expects you to have a lock
-function backupBoxAndApps(callback) {
-    callback = callback || function () { }; // callback can be empty for timer triggered backup
-
-    apps.getAll(function (error, allApps) {
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        var processed = 0;
-        var step = 100/(allApps.length+1);
-
-        progress.set(progress.BACKUP, processed, '');
-
-        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
-            ++processed;
-
-            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                if (error && error.reason !== AppsError.BAD_STATE) {
-                    debugApp(app, 'Unable to backup', error);
-                    return iteratorCallback(error);
-                }
-
-                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
-
-                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
-            });
-        }, function appsBackedUp(error, backupIds) {
-            if (error) {
-                progress.set(progress.BACKUP, 100, error.message);
-                return callback(error);
+        for (var i = 0; i < ifaces[dev].length; i++) {
+            if (ifaces[dev][i].family === 'IPv4') {
+                gCachedIp = ifaces[dev][i].address;
+                return gCachedIp;
             }
+        }
+    }
 
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
+    return null;
+};
 
-            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
-                progress.set(progress.BACKUP, 100, error ? error.message : '');
-                callback(error, restoreKey);
-            });
-        });
+function getStatus(callback) {
+    assert(typeof callback === 'function');
+
+    userdb.count(function (error, count) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        callback(null, { activated: count !== 0, version: config.version() });
     });
 }
+
+function getConfig(callback) {
+    assert(typeof callback === 'function');
+
+    callback(null, {
+        appServerUrl: config.appServerUrl(),
+        isDev: /dev/i.test(config.get('boxVersionsUrl')),
+        fqdn: config.fqdn(),
+        ip: getIp(),
+        version: config.version(),
+        update: updater.getUpdateInfo()
+    })
+}
+
+function sendHeartBeat() {
+    var HEARTBEAT_INTERVAL = 1000 * 60;
+
+    if (!config.appServerUrl()) {
+        debug('No appstore server url set. Not sending heartbeat.');
+        return;
+    }
+
+    if (!config.token()) {
+        debug('No appstore server token set. Not sending heartbeat.');
+        return;
+    }
+
+    var url = config.appServerUrl() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
+    debug('Sending heartbeat ' + url);
+
+    superagent.get(url).query({ token: config.token(), version: config.version() }).end(function (error, result) {
+        if (error) debug('Error sending heartbeat.', error);
+        else if (result.statusCode !== 200) debug('Server responded to heartbeat with ' + result.statusCode);
+        else debug('Heartbeat successful');
+
+        setTimeout(sendHeartBeat, HEARTBEAT_INTERVAL);
+    });
+};
+
+function sendMailDnsRecordsRequest(callback) {
+    assert(typeof callback === 'function');
+
+    var DKIM_SELECTOR = 'mail';
+    var DMARC_REPORT_EMAIL = 'girish@forwardbias.in';
+
+    var dkimPublicKeyFile = path.join(paths.HARAKA_CONFIG_DIR, 'dkim/' + config.fqdn() + '/public');
+    var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
+
+    if (publicKey === null) return callback(new Error('Error reading dkim public key'));
+
+    // remove header, footer and new lines
+    publicKey = publicKey.split('\n').slice(1, -2).join('');
+
+    // note that dmarc requires special DNS records for external RUF and RUA
+    var records = [
+        // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
+        { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + getIp() + ' ~all"' },
+        // t=s limits the domainkey to this domain and not it's subdomains
+        { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', value: '"v=DKIM1; t=s; p=' + publicKey + '"' },
+        // DMARC requires special setup if report email id is in different domain
+        { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
+    ];
+
+    debug('sendMailDnsRecords request:%s', JSON.stringify(records));
+
+    superagent
+        .post(config.appServerUrl() + '/api/v1/subdomains')
+        .set('Accept', 'application/json')
+        .query({ token: config.token() })
+        .send({ records: records })
+        .end(function (error, res) {
+            if (error) return callback(error);
+
+            debug('sendMailDnsRecords status: %s', res.status);
+
+            if (res.status === 409) return callback(null); // already registered
+
+            if (res.status !== 201) return callback(new Error(util.format('Failed to add Mail DNS records: %s %j', res.status, res.body)));
+
+            return callback(null, res.body.ids);
+        });
+}
+
+function addMailDnsRecords() {
+    // TODO assert replaced with a non fatal return, for local development
+    if (!config.token()) return;
+
+    if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
+
+    sendMailDnsRecordsRequest(function (error, ids) {
+        if (error) {
+            console.error('Mail DNS record addition failed', error);
+            gAddMailDnsRecordsTimerId = setTimeout(addMailDnsRecords, 30000);
+            return;
+        }
+
+        debug('Added Mail DNS records successfully');
+        config.set('mailDnsRecordIds', ids);
+    });
+}
+
+function setCertificate(certificate, key, callback) {
+    assert(typeof certificate === 'string');
+    assert(typeof key === 'string');
+
+    debug('Updating certificates');
+
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), certificate)) {
+        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+    }
+
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) {
+        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+    }
+
+    execFile(SUDO, [ RELOAD_NGINX_CMD ], { timeout: 10000 }, function (error) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+

src/collectd.config.ejs

@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.stat">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.stat">
     Instance "<%= appId %>-memory"
     Separator " \\n"
     <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/docker/<%= containerId %>/memory.max_usage_in_bytes">
     Instance "<%= appId %>-memory"
     Separator "\\n"
     <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker/<%= containerId %>/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/docker/<%= containerId %>/cpuacct.stat">
     Instance "<%= appId %>-cpu"
     Separator " \\n"
     <Result>
@@ -30,3 +30,13 @@ LoadPlugin "table"
     </Result>
   </Table>
 </Plugin>
+
+LoadPlugin "filecount"
+<Plugin "filecount">
+  <Directory "/home/yellowtent/data/appdata/<%= appId %>">
+    Instance "<%= appId %>-appdata"
+    IncludeHidden true
+    Recursive true
+  </Directory>
+</Plugin>
+

src/config.js

@@ -1,202 +0,0 @@
-/* jslint node: true */
-
-'use strict';
-
-exports = module.exports = {
-    baseDir: baseDir,
-    dnsInSync: dnsInSync,
-    setDnsInSync: setDnsInSync,
-
-    // values set here will be lost after a upgrade/update. use the sqlite database
-    // for persistent values that need to be backed up
-    get: get,
-    set: set,
-
-    // ifdefs to check environment
-    CLOUDRON: process.env.BOX_ENV === 'cloudron',
-    TEST: process.env.BOX_ENV === 'test',
-
-    // convenience getters
-    provider: provider,
-    apiServerOrigin: apiServerOrigin,
-    webServerOrigin: webServerOrigin,
-    fqdn: fqdn,
-    token: token,
-    version: version,
-    isCustomDomain: isCustomDomain,
-    database: database,
-
-    // these values are derived
-    adminOrigin: adminOrigin,
-    internalAdminOrigin: internalAdminOrigin,
-    adminFqdn: adminFqdn,
-    appFqdn: appFqdn,
-    zoneName: zoneName,
-
-    isDev: isDev,
-
-    // for testing resets to defaults
-    _reset: initConfig
-};
-
-var assert = require('assert'),
-    constants = require('./constants.js'),
-    fs = require('fs'),
-    path = require('path'),
-    safe = require('safetydance'),
-    _ = require('underscore');
-
-var homeDir = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
-
-var data = { };
-
-function baseDir() {
-    if (exports.CLOUDRON) return homeDir;
-    if (exports.TEST) return path.join(homeDir, '.cloudron_test');
-}
-
-var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
-
-function dnsInSync() {
-    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
-}
-
-function setDnsInSync(content) {
-    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
-}
-
-function saveSync() {
-    fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
-}
-
-function initConfig() {
-    // setup defaults
-    data.fqdn = 'localhost';
-
-    data.token = null;
-    data.adminEmail = null;
-    data.boxVersionsUrl = null;
-    data.version = null;
-    data.isCustomDomain = false;
-    data.webServerOrigin = null;
-    data.internalPort = 3001;
-    data.ldapPort = 3002;
-    data.oauthProxyPort = 3003;
-    data.simpleAuthPort = 3004;
-    data.provider = 'caas';
-
-    if (exports.CLOUDRON) {
-        data.port = 3000;
-        data.apiServerOrigin = null;
-        data.database = null;
-    } else if (exports.TEST) {
-        data.port = 5454;
-        data.apiServerOrigin = 'http://localhost:6060'; // hock doesn't support https
-        data.database = {
-            hostname: 'localhost',
-            username: 'root',
-            password: '',
-            port: 3306,
-            name: 'boxtest'
-        };
-        data.token = 'APPSTORE_TOKEN';
-    } else {
-        assert(false, 'Unknown environment. This should not happen!');
-    }
-
-    if (safe.fs.existsSync(cloudronConfigFileName)) {
-        var existingData = safe.JSON.parse(safe.fs.readFileSync(cloudronConfigFileName, 'utf8'));
-        _.extend(data, existingData); // overwrite defaults with saved config
-        return;
-    }
-
-    saveSync();
-}
-
-// cleanup any old config file we have for tests
-if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
-
-initConfig();
-
-// set(obj) or set(key, value)
-function set(key, value) {
-    if (typeof key === 'object') {
-        var obj = key;
-        for (var k in obj) {
-            assert(k in data, 'config.js is missing key "' + k + '"');
-            data[k] = obj[k];
-        }
-    } else {
-        data = safe.set(data, key, value);
-    }
-    saveSync();
-}
-
-function get(key) {
-    assert.strictEqual(typeof key, 'string');
-
-    return safe.query(data, key);
-}
-
-function apiServerOrigin() {
-    return get('apiServerOrigin');
-}
-
-function webServerOrigin() {
-    return get('webServerOrigin');
-}
-
-function fqdn() {
-    return get('fqdn');
-}
-
-// keep this in sync with start.sh admin.conf generation code
-function appFqdn(location) {
-    assert.strictEqual(typeof location, 'string');
-
-    if (location === '') return fqdn();
-    return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
-}
-
-function adminFqdn() {
-    return appFqdn(constants.ADMIN_LOCATION);
-}
-
-function adminOrigin() {
-    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
-}
-
-function internalAdminOrigin() {
-    return 'http://127.0.0.1:' + get('port');
-}
-
-function token() {
-    return get('token');
-}
-
-function version() {
-    return get('version');
-}
-
-function isCustomDomain() {
-    return get('isCustomDomain');
-}
-
-function zoneName() {
-    if (isCustomDomain()) return fqdn(); // the appstore sets up the custom domain as a zone
-
-    // for shared domain name, strip out the hostname
-    return fqdn().substr(fqdn().indexOf('.') + 1);
-}
-
-function database() {
-    return get('database');
-}
-
-function isDev() {
-    return /dev/i.test(get('boxVersionsUrl'));
-}
-
-function provider() {
-    return get('provider');
-}

src/constants.js

@@ -1,12 +0,0 @@
-'use strict';
-
- // default admin installation location. keep in sync with ADMIN_LOCATION in setup/start.sh and BOX_ADMIN_LOCATION in appstore constants.js
-exports = module.exports = {
-    ADMIN_LOCATION: 'my',
-    API_LOCATION: 'api', // this is unused but reserved for future use (#403)
-    ADMIN_NAME: 'Settings',
-
-    ADMIN_CLIENT_ID: 'webadmin', // oauth client id
-    ADMIN_APPID: 'admin' // admin appid (settingsdb)
-};
-

src/cron.js

@@ -1,196 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize
-};
-
-var apps = require('./apps.js'),
-    assert = require('assert'),
-    certificates = require('./certificates.js'),
-    cloudron = require('./cloudron.js'),
-    config = require('./config.js'),
-    CronJob = require('cron').CronJob,
-    debug = require('debug')('box:cron'),
-    janitor = require('./janitor.js'),
-    scheduler = require('./scheduler.js'),
-    settings = require('./settings.js'),
-    updateChecker = require('./updatechecker.js');
-
-var gAutoupdaterJob = null,
-    gBoxUpdateCheckerJob = null,
-    gAppUpdateCheckerJob = null,
-    gHeartbeatJob = null,
-    gBackupJob = null,
-    gCleanupTokensJob = null,
-    gDockerVolumeCleanerJob = null,
-    gSchedulerSyncJob = null,
-    gCertificateRenewJob = null;
-
-var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
-
-// cron format
-// Seconds: 0-59
-// Minutes: 0-59
-// Hours: 0-23
-// Day of Month: 1-31
-// Months: 0-11
-// Day of Week: 0-6
-
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    gHeartbeatJob = new CronJob({
-        cronTime: '00 */1 * * * *', // every minute
-        onTick: cloudron.sendHeartbeat,
-        start: true
-    });
-    cloudron.sendHeartbeat(); // latest unpublished version of CronJob has runOnInit
-
-    if (cloudron.isConfiguredSync()) {
-        recreateJobs(callback);
-    } else {
-        cloudron.events.on(cloudron.EVENT_ACTIVATED, recreateJobs);
-        callback();
-    }
-}
-
-function recreateJobs(unusedTimeZone, callback) {
-    if (typeof unusedTimeZone === 'function') callback = unusedTimeZone;
-
-    settings.getAll(function (error, allSettings) {
-        debug('Creating jobs with timezone %s', allSettings[settings.TIME_ZONE_KEY]);
-
-        if (gBackupJob) gBackupJob.stop();
-        gBackupJob = new CronJob({
-            cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: cloudron.ensureBackup,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
-        gBoxUpdateCheckerJob = new CronJob({
-            cronTime: '00 */10 * * * *', // every 10 minutes
-            onTick: updateChecker.checkBoxUpdates,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
-        gAppUpdateCheckerJob = new CronJob({
-            cronTime: '00 */10 * * * *', // every 10 minutes
-            onTick: updateChecker.checkAppUpdates,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        if (gCleanupTokensJob) gCleanupTokensJob.stop();
-        gCleanupTokensJob = new CronJob({
-            cronTime: '00 */30 * * * *', // every 30 minutes
-            onTick: janitor.cleanupTokens,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
-        gDockerVolumeCleanerJob = new CronJob({
-            cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: janitor.cleanupDockerVolumes,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
-        gSchedulerSyncJob = new CronJob({
-            cronTime: config.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
-            onTick: scheduler.sync,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        if (gCertificateRenewJob) gCertificateRenewJob.stop();
-        gCertificateRenewJob = new CronJob({
-            cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: certificates.autoRenew,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
-
-        settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
-        settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
-        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY]);
-
-        settings.events.removeListener(settings.TIME_ZONE_KEY, recreateJobs);
-        settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
-
-        if (callback) callback();
-    });
-}
-
-function autoupdatePatternChanged(pattern) {
-    assert.strictEqual(typeof pattern, 'string');
-    assert(gBoxUpdateCheckerJob);
-
-    debug('Auto update pattern changed to %s', pattern);
-
-    if (gAutoupdaterJob) gAutoupdaterJob.stop();
-
-    if (pattern === 'never') return;
-
-    gAutoupdaterJob = new CronJob({
-        cronTime: pattern,
-        onTick: function() {
-            var updateInfo = updateChecker.getUpdateInfo();
-            if (updateInfo.box) {
-                debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
-            } else if (updateInfo.apps) {
-                debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
-            } else {
-                debug('No auto updates available');
-            }
-        },
-        start: true,
-        timeZone: gBoxUpdateCheckerJob.cronTime.zone // hack
-    });
-}
-
-function uninitialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    cloudron.events.removeListener(cloudron.EVENT_ACTIVATED, recreateJobs);
-
-    settings.events.removeListener(settings.TIME_ZONE_KEY, recreateJobs);
-    settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
-
-    if (gAutoupdaterJob) gAutoupdaterJob.stop();
-    gAutoupdaterJob = null;
-
-    if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
-    gBoxUpdateCheckerJob = null;
-
-    if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
-    gAppUpdateCheckerJob = null;
-
-    if (gHeartbeatJob) gHeartbeatJob.stop();
-    gHeartbeatJob = null;
-
-    if (gBackupJob) gBackupJob.stop();
-    gBackupJob = null;
-
-    if (gCleanupTokensJob) gCleanupTokensJob.stop();
-    gCleanupTokensJob = null;
-
-    if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
-    gDockerVolumeCleanerJob = null;
-
-    if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
-    gSchedulerSyncJob = null;
-
-    if (gCertificateRenewJob) gCertificateRenewJob.stop();
-    gCertificateRenewJob = null;
-
-    callback();
-}

src/database.js

@@ -1,204 +1,123 @@
-/* jslint node: true */
+/* jslint node:true */
 
 'use strict';
 
+var assert = require('assert'),
+    async = require('async'),
+    config = require('../config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:database'),
+    paths = require('./paths.js'),
+    sqlite3 = require('sqlite3');
+
 exports = module.exports = {
     initialize: initialize,
     uninitialize: uninitialize,
-    query: query,
-    transaction: transaction,
-
+    removePrivates: removePrivates,
     beginTransaction: beginTransaction,
     rollback: rollback,
     commit: commit,
+    clear: clear,
 
-    _clear: clear
+    get: get,
+    all: all,
+    run: run
 };
 
-var assert = require('assert'),
-    async = require('async'),
-    once = require('once'),
-    config = require('./config.js'),
-    mysql = require('mysql'),
-    util = require('util');
+var gConnectionPool = [ ], // used to track active transactions
+    gDatabase = null;
 
-var gConnectionPool = null,
-    gDefaultConnection = null;
+var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 
-function initialize(options, callback) {
-    if (typeof options === 'function') {
-        callback = options;
-        options = {
-            connectionLimit: 5
-        };
-    }
-
-    assert.strictEqual(typeof options.connectionLimit, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (gConnectionPool !== null) return callback(null);
-
-    gConnectionPool  = mysql.createPool({
-        connectionLimit: options.connectionLimit,
-        host: config.database().hostname,
-        user: config.database().username,
-        password: config.database().password,
-        port: config.database().port,
-        database: config.database().name,
-        multipleStatements: false,
-        ssl: false
+function initialize(callback) {
+    gDatabase = new sqlite3.Database(paths.DATABASE_FILENAME);
+    gDatabase.on('error', function (error) {
+        console.error('Database error in ' + paths.DATABASE_FILENAME + ':', error);
     });
 
-    reconnect(callback);
+    gDatabase.run('PRAGMA busy_timeout=5000', callback);
 }
 
 function uninitialize(callback) {
-    if (gConnectionPool) {
-        gConnectionPool.end(callback);
-        gConnectionPool = null;
-    } else {
-        callback(null);
-    }
-}
+    assert(typeof callback === 'function');
 
-function setupConnection(connection, callback) {
-    assert.strictEqual(typeof connection, 'object');
-    assert.strictEqual(typeof callback, 'function');
+    debug('Closing database');
+    gDatabase.close();
+    gDatabase = null;
 
-    connection.on('error', console.error);
+    debug('Closing %d active transactions', gConnectionPool.length);
+    gConnectionPool.forEach(function (conn) { conn.close(); });
+    gConnectionPool = [ ];
 
-    async.series([
-        connection.query.bind(connection, 'USE ' + config.database().name),
-        connection.query.bind(connection, 'SET SESSION sql_mode = \'strict_all_tables\'')
-    ], function (error) {
-        connection.removeListener('error', console.error);
-
-        if (error) connection.release();
-
-        callback(error);
-    });
-}
-
-function reconnect(callback) {
-    callback = callback ? once(callback) : function () {};
-
-    gConnectionPool.getConnection(function (error, connection) {
-        if (error) {
-            console.error('Unable to reestablish connection to database. Try again in a bit.', error.message);
-            return setTimeout(reconnect.bind(null, callback), 1000);
-        }
-
-        connection.on('error', function (error) {
-            // by design, we catch all normal errors by providing callbacks.
-            // this function should be invoked only when we have no callbacks pending and we have a fatal error
-            assert(error.fatal, 'Non-fatal error on connection object');
-
-            console.error('Unhandled mysql connection error.', error);
-
-            // This is most likely an issue an can cause double callbacks from reconnect()
-            setTimeout(reconnect.bind(null, callback), 1000);
-        });
-
-        setupConnection(connection, function (error) {
-            if (error) return setTimeout(reconnect.bind(null, callback), 1000);
-
-            gDefaultConnection = connection;
-
-            callback(null);
-        });
-    });
+    callback(null);
 }
 
 function clear(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    // the clear funcs don't completely clear the db, they leave the migration code defaults
     async.series([
-        require('./appdb.js')._clear,
-        require('./authcodedb.js')._clear,
-        require('./clientdb.js')._clear,
-        require('./tokendb.js')._clear,
-        require('./userdb.js')._clear,
-        require('./settingsdb.js')._clear
+        require('./appdb.js').clear,
+        require('./authcodedb.js').clear,
+        require('./clientdb.js').clear,
+        require('./tokendb.js').clear,
+        require('./userdb.js').clear
     ], callback);
 }
 
-function beginTransaction(callback) {
-    assert.strictEqual(typeof callback, 'function');
+function beginTransaction() {
+    var conn = new sqlite3.Database(paths.DATABASE_FILENAME);
+    conn._started = Date.now();
+    conn._slowWarningIntervalId = setInterval((function () {
+        debug('Transaction running for %d msecs', Date.now() - this._started);
+    }).bind(conn), 2000);
 
-    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
-
-    gConnectionPool.getConnection(function (error, connection) {
-        if (error) return callback(error);
-
-        setupConnection(connection, function (error) {
-            if (error) return callback(error);
-
-            connection.beginTransaction(function (error) {
-                if (error) return callback(error);
-
-                return callback(null, connection);
-            });
-        });
-    });
+    gConnectionPool.push(conn);
+    conn.serialize();
+    conn.run('PRAGMA busy_timeout=5000', NOOP_CALLBACK);
+    conn.run('BEGIN TRANSACTION', NOOP_CALLBACK);
+    return conn;
 }
 
-function rollback(connection, callback) {
-    assert.strictEqual(typeof callback, 'function');
+function rollback(conn, callback) {
+    gConnectionPool.splice(gConnectionPool.indexOf(conn), 1);
+    conn.run('ROLLBACK', NOOP_CALLBACK);
+    clearInterval(conn._slowWarningIntervalId);
+    debug('Transaction took %d msecs', Date.now() - conn._started);
+    conn.close(); // close waits for pending statements
+    if (callback) callback();
+}
 
-    connection.rollback(function (error) {
-        if (error) console.error(error); // can this happen?
+function commit(conn, callback) {
+    gConnectionPool.splice(gConnectionPool.indexOf(conn), 1);
+    conn.run('COMMIT', function (error) {
+        clearInterval(conn._slowWarningIntervalId);
+        debug('Transaction took %d msecs', Date.now() - conn._started);
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        connection.release();
         callback(null);
     });
+    conn.close(); // close waits for pending statements
 }
 
-// FIXME: if commit fails, is it supposed to return an error ?
-function commit(connection, callback) {
-    assert.strictEqual(typeof callback, 'function');
+function removePrivates(obj) {
+    var res = { };
 
-    connection.commit(function (error) {
-        if (error) return rollback(connection, callback);
+    for (var p in obj) {
+        if (!obj.hasOwnProperty(p)) continue;
+        if (p.substring(0, 1) === '_') continue;
+        res[p] = obj[p]; // ## make deep copy?
+    }
 
-        connection.release();
-        return callback(null);
-    });
+    return res;
 }
 
-function query() {
-    var args = Array.prototype.slice.call(arguments);
-    var callback = args[args.length - 1];
-    assert.strictEqual(typeof callback, 'function');
-
-    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
-
-    args[args.length -1 ] = function (error, result) {
-        if (error && error.fatal) {
-            gDefaultConnection = null;
-            setTimeout(reconnect, 1000);
-        }
-
-        callback(error, result);
-    };
-
-    gDefaultConnection.query.apply(gDefaultConnection, args);
+function get() {
+    return gDatabase.get.apply(gDatabase, arguments);
 }
 
-function transaction(queries, callback) {
-    assert(util.isArray(queries));
-    assert.strictEqual(typeof callback, 'function');
-
-    beginTransaction(function (error, conn) {
-        if (error) return callback(error);
-
-        async.mapSeries(queries, function iterator(query, done) {
-            conn.query(query.query, query.args, done);
-        }, function seriesDone(error, results) {
-            if (error) return rollback(conn, callback.bind(null, error));
-
-            commit(conn, callback.bind(null, null, results));
-        });
-    });
+function all() {
+    return gDatabase.all.apply(gDatabase, arguments);
+}
+
+function run() {
+    return gDatabase.run.apply(gDatabase, arguments);
 }
 

src/databaseerror.js

@@ -2,20 +2,20 @@
 
 'use strict';
 
-exports = module.exports = DatabaseError;
-
 var assert = require('assert'),
     util = require('util');
 
+module.exports = exports = DatabaseError;
+
 function DatabaseError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined' || errorOrMessage === null);
+    assert(typeof reason === 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
 
     Error.call(this);
     Error.captureStackTrace(this, this.constructor);
 
     this.reason = reason;
-    if (typeof errorOrMessage === 'undefined' || errorOrMessage === null) {
+    if (typeof errorOrMessage === 'undefined') {
         this.message = reason;
     } else if (typeof errorOrMessage === 'string') {
         this.message = errorOrMessage;
@@ -29,4 +29,5 @@ util.inherits(DatabaseError, Error);
 DatabaseError.INTERNAL_ERROR = 'Internal error';
 DatabaseError.ALREADY_EXISTS = 'Entry already exist';
 DatabaseError.NOT_FOUND = 'Record not found';
-DatabaseError.BAD_FIELD = 'Invalid field';
+DatabaseError.RECORD_SCHEMA = 'Record does not match the schema';
+DatabaseError.FIELD_ERROR = 'Invalid field';

src/developer.js

@@ -1,91 +0,0 @@
-/* jslint node: true */
-
-'use strict';
-
-exports = module.exports = {
-    DeveloperError: DeveloperError,
-
-    enabled: enabled,
-    setEnabled: setEnabled,
-    issueDeveloperToken: issueDeveloperToken,
-    getNonApprovedApps: getNonApprovedApps
-};
-
-var assert = require('assert'),
-    config = require('./config.js'),
-    debug = require('debug')('box:developer'),
-    tokendb = require('./tokendb.js'),
-    settings = require('./settings.js'),
-    superagent = require('superagent'),
-    util = require('util');
-
-function DeveloperError(reason, errorOrMessage) {
-    assert.strictEqual(typeof reason, 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
-
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
-        this.message = reason;
-    } else if (typeof errorOrMessage === 'string') {
-        this.message = errorOrMessage;
-    } else {
-        this.message = 'Internal error';
-        this.nestedError = errorOrMessage;
-    }
-}
-util.inherits(DeveloperError, Error);
-DeveloperError.INTERNAL_ERROR = 'Internal Error';
-DeveloperError.EXTERNAL_ERROR = 'External Error';
-
-function enabled(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.getDeveloperMode(function (error, enabled) {
-        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
-        callback(null, enabled);
-    });
-}
-
-function setEnabled(enabled, callback) {
-    assert.strictEqual(typeof enabled, 'boolean');
-    assert.strictEqual(typeof callback, 'function');
-
-    settings.setDeveloperMode(enabled, function (error) {
-        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
-        callback(null);
-    });
-}
-
-function issueDeveloperToken(user, callback) {
-    assert.strictEqual(typeof user, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    var token = tokendb.generateToken();
-    var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
-
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
-        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
-
-        callback(null, { token: token, expiresAt: expiresAt });
-    });
-}
-
-function getNonApprovedApps(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
-    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
-        if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
-        if (result.statusCode === 401) {
-            debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);
-            return callback(null, []);
-        }
-        if (result.statusCode !== 200) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
-
-        callback(null, result.body.apps || []);
-    });
-}