addons can be null in apps.backupApp

addons.backup already takes care of null.

a future commit will give defaults for all non-default manifest fields
at some point and document them as so

.gitattributes

@@ -1,10 +1,6 @@
 # Skip files when using git archive
 .gitattributes export-ignore
 .gitignore export-ignore
-/release export-ignore
 /scripts export-ignore
 test export-ignore
-/webadmin/src export-ignore
-/webadmin/deploymentConfig.json export-ignore
-/gulpfile.json export-ignore
 

.gitignore

@@ -1,6 +1,8 @@
 node_modules/
 coverage/
 docs/
+webadmin/dist/
+setup/splash/website/
 
 # vim swam files
 *.swp
@@ -9,12 +11,3 @@ docs/
 supervisord.pid
 supervisord.log
 
-# nginx
-nginx/*.log
-nginx/*.pid
-nginx/naked_domain.conf
-nginx/applications/
-
-# release files
-release/versions-dev.json
-

README.md

@@ -9,9 +9,3 @@ Development setup
 ** All apps will be installed as hypened-subdomains of localhost. You should add
    hyphened-subdomains of your apps into /etc/hosts
 
-Running
--------
-* `./run.sh` - this starts up nginx to serve up the webadmin
-* `DEBUG=box:* ./app.js` - this the main box code.
-* Navigate to https://admin-localhost
-

app.js

@@ -5,17 +5,20 @@
 require('supererror')({ splatchError: true });
 
 var server = require('./src/server.js'),
-    config = require('./config.js');
+    ldap = require('./src/ldap.js'),
+    config = require('./src/config.js');
 
 console.log();
 console.log('==========================================');
 console.log(' Cloudron will use the following settings ');
 console.log('==========================================');
 console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : (config.LOCAL ? 'LOCAL' : 'TEST'));
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
+console.log(' Version:                        ', config.version());
 console.log(' Admin Origin:                   ', config.adminOrigin());
 console.log(' Appstore token:                 ', config.token());
-console.log(' Appstore server origin:         ', config.appServerUrl());
+console.log(' Appstore API server origin:     ', config.apiServerOrigin());
+console.log(' Appstore Web server origin:     ', config.webServerOrigin());
 console.log();
 console.log('==========================================');
 console.log();
@@ -27,6 +30,15 @@ server.start(function (err) {
     }
 
     console.log('Server listening on port ' + config.get('port'));
+
+    ldap.start(function (error) {
+        if (error) {
+            console.error('Error LDAP starting server', err);
+            process.exit(1);
+        }
+
+        console.log('LDAP server listen on port ' + config.get('ldapPort'));
+    });
 });
 
 var NOOP_CALLBACK = function () { };

apphealthtask.js

@@ -12,95 +12,104 @@ var appdb = require('./src/appdb.js'),
     debug = require('debug')('box:apphealthtask'),
     docker = require('./src/docker.js'),
     mailer = require('./src/mailer.js'),
-    os = require('os'),
-    superagent = require('superagent');
+    superagent = require('superagent'),
+    util = require('util');
 
 exports = module.exports = {
-    initialize: initialize,
     run: run
 };
 
-var FATAL_CALLBACK = function (error) {
-    if (!error) return;
-    console.error(error);
-    process.exit(2);
-};
+var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
+var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var gHealthInfo = { }; // { time, emailSent }
 
-var HEALTHCHECK_INTERVAL = 30000;
-var gLastSeen = { }; // { time, emailSent }
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
 
 function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     async.series([
         database.initialize,
         mailer.initialize
     ], callback);
 }
 
-function setHealth(app, alive, runState, callback) {
-    assert(typeof app === 'object');
-    assert(typeof alive === 'boolean');
-    assert(typeof runState === 'string');
-    assert(typeof callback === 'function');
+function setHealth(app, health, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    var healthy = true; // app is unhealthy if not alive for 2 mins
     var now = new Date();
 
-    if (alive || !(app.id in gLastSeen)) { // give never seen apps 2 mins to come up
-        gLastSeen[app.id] = { time: now, emailSent: false };
-    } else if (Math.abs(now - gLastSeen[app.id].time) > 120 * 1000) { // not seen for 2 mins
-        debug('app %s not seen for more than 2 mins, marking as unhealthy', app.id);
-        healthy = false;
+    if (!(app.id in gHealthInfo)) { // add new apps to list
+        gHealthInfo[app.id] = { time: now, emailSent: false };
     }
 
-    if (!healthy && !gLastSeen[app.id].emailSent) {
-        gLastSeen[app.id].emailSent = true;
+    if (health === appdb.HEALTH_HEALTHY) {
+        gHealthInfo[app.id].time = now;
+    } else if (Math.abs(now - gHealthInfo[app.id].time) > UNHEALTHY_THRESHOLD) {
+        if (gHealthInfo[app.id].emailSent) return callback(null);
+
+        debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
+
         mailer.appDied(app);
+        gHealthInfo[app.id].emailSent = true;
+    } else {
+        debugApp(app, 'waiting for sometime to update the app health');
+        return callback(null);
     }
 
-    appdb.setHealth(app.id, healthy, runState, function (error) {
+    appdb.setHealth(app.id, health, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled?
         if (error) return callback(error);
 
-        app.healthy = healthy;
-        app.runState = runState;
+        app.health = health;
 
         callback(null);
     });
 }
 
 
-// # TODO should probably poll from the outside network instead of the docker network?
 // callback is called with error for fatal errors and not if health check failed
 function checkAppHealth(app, callback) {
-    // only check status of installed apps. we could possibly optimize more by checking runState as well
-    if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(null);
+    if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
+        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
+        return callback(null);
+    }
 
     var container = docker.getContainer(app.containerId),
         manifest = app.manifest;
 
     container.inspect(function (err, data) {
         if (err || !data || !data.State) {
-            debug('Error inspecting container');
-            return setHealth(app, false, appdb.RSTATE_ERROR, callback);
+            debugApp(app, 'Error inspecting container');
+            return setHealth(app, appdb.HEALTH_ERROR, callback);
         }
 
         if (data.State.Running !== true) {
-            debug('app %s has exited', app.id);
-            return setHealth(app, false, appdb.RSTATE_DEAD, callback);
+            debugApp(app, 'exited');
+            return setHealth(app, appdb.HEALTH_DEAD, callback);
         }
 
+        // poll through docker network instead of nginx to bypass any potential oauth proxy
         var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
         superagent
             .get(healthCheckUrl)
+            .redirects(0)
             .timeout(HEALTHCHECK_INTERVAL)
             .end(function (error, res) {
 
-            if (error || res.status !== 200) {
-                debug('app %s is not alive ', app.id);
-                setHealth(app, false, appdb.RSTATE_RUNNING, callback);
+            if (error || res.status >= 400) { // 2xx and 3xx are ok
+                debugApp(app, 'not alive : %s', error || res.status);
+                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
             } else {
-                debug('app %s is alive', app.id);
-                setHealth(app, true, appdb.RSTATE_RUNNING, callback);
+                debugApp(app, 'alive');
+                setHealth(app, appdb.HEALTH_HEALTHY, callback);
             }
         });
     });
@@ -117,19 +126,22 @@ function processApps(callback) {
     });
 }
 
-function run(callback) {
+function run() {
     processApps(function (error) {
-        if (error) return callback(error);
-        setTimeout(run.bind(null, callback), HEALTHCHECK_INTERVAL);
+        if (error) console.error(error);
+
+        setTimeout(run, HEALTHCHECK_INTERVAL);
     });
 }
 
 if (require.main === module) {
-    initialize();
+    initialize(function (error) {
+        if (error) {
+            console.error('apphealth task exiting with error', error);
+            process.exit(1);
+        }
 
-    run(function (error) {
-        console.error('apphealth task exiting with error.', error);
-        process.exit(error ? 1 : 0);
+        run();
     });
 }
 

crashnotifier.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+'use strict';
+
+// WARNING This is a supervisor eventlistener!
+//         The communication happens via stdin/stdout
+//         !! No console.log() allowed
+//         !! Do not set DEBUG
+
+var assert = require('assert'),
+    mailer = require('./src/mailer.js'),
+    safe = require('safetydance'),
+    supervisor = require('supervisord-eventlistener'),
+    path = require('path'),
+    util = require('util');
+
+var gLastNotifyTime = {};
+var gCooldownTime = 1000 * 60  * 5; // 5 min
+var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
+
+function collectLogs(program, callback) {
+    assert.strictEqual(typeof program, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+supervisor.on('PROCESS_STATE_EXITED', function (headers, data) {
+    if (data.expected === '1') return console.error('Normal app %s exit', data.processname);
+
+    console.error('%s exited unexpectedly', data.processname);
+
+    collectLogs(data.processname, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        if (!gLastNotifyTime[data.processname] || gLastNotifyTime[data.processname] < Date.now() - gCooldownTime) {
+            console.error('Send mail.');
+            mailer.sendCrashNotification(data.processname, result);
+            gLastNotifyTime[data.processname] = Date.now();
+        } else {
+            console.error('Do not send mail, already sent one recently.');
+        }
+    });
+});
+
+mailer.initialize(function () {
+    supervisor.listen(process.stdin, process.stdout);
+    console.error('Crashnotifier listening...');
+});

gulpfile.js

@@ -2,73 +2,158 @@
 
 'use strict';
 
-var _ejs = require('ejs'),
-    ejs = require('gulp-ejs'),
+var ejs = require('gulp-ejs'),
     gulp = require('gulp'),
     del = require('del'),
-    path = require('path'),
     concat = require('gulp-concat'),
+    uglify = require('gulp-uglify'),
+    serve = require('gulp-serve'),
+    sass = require('gulp-sass'),
     sourcemaps = require('gulp-sourcemaps'),
-    fs = require('fs');
-
-_ejs.filters.basename = function (obj) {
-    return path.basename(obj);
-};
+    minifyCSS = require('gulp-minify-css'),
+    autoprefixer = require('gulp-autoprefixer'),
+    argv = require('yargs').argv;
 
 gulp.task('3rdparty', function () {
-    return gulp.src([
-            'webadmin/src/3rdparty/**/*.js',
-            'webadmin/src/3rdparty/**/*.css',
-            'webadmin/src/3rdparty/**/*.otf',
-            'webadmin/src/3rdparty/**/*.eot',
-            'webadmin/src/3rdparty/**/*.svg',
-            'webadmin/src/3rdparty/**/*.ttf',
-            'webadmin/src/3rdparty/**/*.woff',
-            'webadmin/src/3rdparty/**/*.js'
+    gulp.src([
+        'webadmin/src/3rdparty/**/*.js',
+        'webadmin/src/3rdparty/**/*.map',
+        'webadmin/src/3rdparty/**/*.css',
+        'webadmin/src/3rdparty/**/*.otf',
+        'webadmin/src/3rdparty/**/*.eot',
+        'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.ttf',
+        'webadmin/src/3rdparty/**/*.woff',
+        'webadmin/src/3rdparty/**/*.woff2'
         ])
-        .pipe(gulp.dest('webadmin/dist/3rdparty/'));
+        .pipe(gulp.dest('webadmin/dist/3rdparty/'))
+        .pipe(gulp.dest('setup/splash/website/3rdparty'));
+
+    gulp.src('node_modules/bootstrap-sass/assets/javascripts/bootstrap.min.js')
+        .pipe(gulp.dest('webadmin/dist/3rdparty/js'))
+        .pipe(gulp.dest('setup/splash/website/3rdparty/js'));
 });
 
+
+// --------------
+// JavaScript
+// --------------
+
+gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
+
+var oauth = {
+    clientId: argv.clientId || 'cid-webadmin',
+    clientSecret: argv.clientSecret || 'unused',
+    apiOrigin: argv.apiOrigin || ''
+};
+
+console.log();
+console.log('Using OAuth credentials:');
+console.log(' ClientId:     %s', oauth.clientId);
+console.log(' ClientSecret: %s', oauth.clientSecret);
+console.log(' Cloudron API: %s', oauth.apiOrigin || 'default');
+console.log();
+
 gulp.task('js-index', function () {
-    return gulp.src(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'])
+    gulp.src([
+        'webadmin/src/js/index.js',
+        'webadmin/src/js/client.js',
+        'webadmin/src/js/appstore.js',
+        'webadmin/src/js/main.js',
+        'webadmin/src/views/*.js'
+        ])
+        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
         .pipe(sourcemaps.init())
-        .pipe(concat('index.js'))
+        .pipe(concat('index.js', { newLine: ';' }))
+        .pipe(uglify())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
 gulp.task('js-setup', function () {
-    return gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
+    gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
+        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
         .pipe(sourcemaps.init())
-        .pipe(concat('setup.js'))
+        .pipe(concat('setup.js', { newLine: ';' }))
+        .pipe(uglify())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('js', ['js-index', 'js-setup'], function () {});
-
-gulp.task('htmlViews', function () {
-    return gulp.src('webadmin/src/views/*.html')
-        .pipe(gulp.dest('webadmin/dist/views'));
+gulp.task('js-error', function () {
+    gulp.src(['webadmin/src/js/error.js'])
+        .pipe(sourcemaps.init())
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('html_templates', function () {
-    var config = JSON.parse(fs.readFileSync('./webadmin/deploymentConfig.json'));
-
-    return gulp.src('webadmin/src/*.ejs')
-        .pipe(ejs(config, { ext: '.html' }))
-        .pipe(gulp.dest('webadmin/dist'));
+gulp.task('js-update', function () {
+    gulp.src(['webadmin/src/js/update.js'])
+        .pipe(sourcemaps.init())
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'))
+        .pipe(gulp.dest('setup/splash/website/js'));
 });
 
-gulp.task('html', ['html_templates', 'htmlViews'], function () {
-    return gulp.src('webadmin/src/*.html')
-        .pipe(gulp.dest('webadmin/dist'));
+
+// --------------
+// HTML
+// --------------
+
+gulp.task('html', ['html-views', 'html-update'], function () {
+    return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
 });
 
-gulp.task('clean', function (callback) {
-    del(['webadmin/dist'], callback);
+gulp.task('html-update', function () {
+    return gulp.src(['webadmin/src/update.html']).pipe(gulp.dest('setup/splash/website'));
 });
 
-gulp.task('default', ['clean'], function () {
-    gulp.start('html', 'js', '3rdparty');
+gulp.task('html-views', function () {
+    return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
 });
+
+// --------------
+// CSS
+// --------------
+
+gulp.task('css', function () {
+    return gulp.src('webadmin/src/*.scss')
+        .pipe(sourcemaps.init())
+        .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
+        .pipe(autoprefixer())
+        .pipe(minifyCSS())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist'))
+        .pipe(gulp.dest('setup/splash/website'));
+});
+
+gulp.task('images', function () {
+    return gulp.src('webadmin/src/img/**')
+        .pipe(gulp.dest('webadmin/dist/img'));
+});
+
+// --------------
+// Utilities
+// --------------
+
+gulp.task('watch', ['default'], function () {
+    gulp.watch(['webadmin/src/*.scss'], ['css']);
+    gulp.watch(['webadmin/src/img/*'], ['images']);
+    gulp.watch(['webadmin/src/**/*.html'], ['html']);
+    gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
+    gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
+    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
+    gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
+    gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
+    gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);
+});
+
+gulp.task('clean', function () {
+    del.sync(['webadmin/dist', 'setup/splash/website']);
+});
+
+gulp.task('default', ['clean', 'html', 'js', '3rdparty', 'images', 'css'], function () {});
+
+gulp.task('develop', ['watch'], serve({ root: 'webadmin/dist', port: 4000 }));

janitor.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+var assert = require('assert'),
+    debug = require('debug')('box:janitor'),
+    async = require('async'),
+    tokendb = require('./src/tokendb.js'),
+    authcodedb = require('./src/authcodedb.js'),
+    database = require('./src/database.js');
+
+var TOKEN_CLEANUP_INTERVAL = 30000;
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        database.initialize
+    ], callback);
+}
+
+function cleanupExpiredTokens(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired tokens.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupExpiredAuthCodes(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    authcodedb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired authcodes.', result);
+
+        callback(null);
+    });
+}
+
+function run() {
+    cleanupExpiredTokens(function (error) {
+        if (error) console.error(error);
+
+        cleanupExpiredAuthCodes(function (error) {
+            if (error) console.error(error);
+
+            setTimeout(run, TOKEN_CLEANUP_INTERVAL);
+        });
+    });
+}
+
+if (require.main === module) {
+    initialize(function (error) {
+        if (error) {
+            console.error('janitor task exiting with error', error);
+            process.exit(1);
+        }
+
+        run();
+    });
+}
+

migrations/20141021192552-db-create.js

@@ -1,8 +1,12 @@
 var dbm = require('db-migrate');
 var type = dbm.dataType;
+var url = require('url');
 
 exports.up = function(db, callback) {
-    callback();
+    var dbName = url.parse(process.env.DATABASE_URL).path.substr(1); // remove slash
+
+    // by default, mysql collates case insensitively. 'utf8_general_cs' is not available
+    db.runSql('ALTER DATABASE ' + dbName + '  DEFAULT CHARACTER SET=utf8 DEFAULT COLLATE utf8_bin', callback);
 };
 
 exports.down = function(db, callback) {

migrations/20141021235251-clients-add-webadmin.js

@@ -1,21 +0,0 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
-
-var uuid = require('node-uuid');
-
-exports.up = function(db, callback) {
-    var scopes = 'root,profile,users,apps,settings,roleAdmin';
-    var adminOrigin = 'https://admin-localhost';
-
-    // postinstall.sh creates the webadmin entry in production mode
-    if (process.env.NODE_ENV !== 'test') return callback(null);
-
-    db.runSql('INSERT INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) ' +
-              'VALUES (?, ?, ?, ?, ?, ?, ?)', [ uuid.v4(), 'webadmin', 'cid-webadmin', 'unused', 'WebAdmin', adminOrigin, scopes ],
-              callback);
-};
-
-exports.down = function(db, callback) {
-    // not sure what is meaningful here
-    callback(null);
-};

migrations/20141022153739-settings-add-naked-domain.js

@@ -1,10 +0,0 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('INSERT INTO settings (key, value) VALUES (?, ?)', [ 'naked_domain', null ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DELETE FROM settings WHERE key=?', [ 'naked_domain' ], callback);
-};

migrations/20141128175100-add-addonconfigs.js

@@ -1,15 +0,0 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('CREATE TABLE appAddonConfigs(' +
-              ' appId VARCHAR(512) NOT NULL,' +
-              ' addonId VARCHAR(32) NOT NULL,' +
-              ' value VARCHAR(512) NOT NULL,' +
-              ' FOREIGN KEY(appId) REFERENCES apps(id))', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DROP TABLE appAddonConfigs', callback);
-};
-

migrations/20150303114527-users-add-resetToken.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN resetToken VARCHAR(128) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN resetToken', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150303160528-tokens-expires-bigint.js

@@ -0,0 +1,20 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM tokens', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE tokens MODIFY expires BIGINT', [], function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens MODIFY expires VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150303171203-authcodes-add-expiresAt.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE authcodes ADD COLUMN expiresAt BIGINT NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE authcodes DROP COLUMN expiresAt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150308054950-appportbindings-add-environmentVariable.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings ADD COLUMN environmentVariable VARCHAR(128) NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings DROP COLUMN environmentVariable', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150309023251-appportbindings-drop-containerport.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings DROP COLUMN containerPort', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings ADD COLUMN containerPort VARCHAR(5) NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150311120713-tokens-rename-userid-to-identifier.js

@@ -0,0 +1,20 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM tokens', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE tokens CHANGE userId identifier VARCHAR(128) NOT NULL', [], function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens CHANGE identifier userId VARCHAR(128) NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150323044254-apps-drop-version.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN version', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN version VARCHAR(32)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150323174906-apps-alter-health.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN healthy, ADD COLUMN health VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN health, ADD COLUMN healthy INTEGER', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150326072904-apps-add-lastBackupId.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupId VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupId', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150404093025-apps-add-createdAt.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN createdAt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150430165335-settings-default-autoupdatepattern.js

@@ -0,0 +1,12 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    // everyday at 1am
+    db.runSql('INSERT settings (name, value) VALUES("autoupdate_pattern", ?)', [ '00 00 1 * * *' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="autoupdate_pattern"', [ ], callback);
+}
+

migrations/20150430210225-settings-default-timezone.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+var safe = require('safetydance');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
+    tz = tz ? tz.trim() : 'America/Los_Angeles';
+
+    db.runSql('INSERT settings (name, value) VALUES("time_zone", ?)', [ tz ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="time_zone"', [ ], callback);
+};
+

migrations/20150615075901-users-add-unique-constraints.js

@@ -0,0 +1,24 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+
+    // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address
+
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_username UNIQUE (username)'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_email UNIQUE (email)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_username'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(512)'),
+        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_email'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(512)'),
+    ], callback);
+};

migrations/20150615134751-users-adjust-username-and-email-constraints.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
+    ], callback);
+};

migrations/20150618192028-apps-add-lastManifestJson.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastManifestJson VARCHAR(2048)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastManifestJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150710044740-apps-rename-lastBackupConfigJson.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastManifestJson lastBackupConfigJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastBackupConfigJson lastManifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150710170847-apps-add-oldConfigJson.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oldConfigJson VARCHAR(2048)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oldConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150719014338-settings-remove-all.js

@@ -0,0 +1,10 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM settings', [ ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/initial-schema.sql

@@ -2,64 +2,66 @@ CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
     username VARCHAR(512) NOT NULL,
     email VARCHAR(512) NOT NULL,
-    _password VARCHAR(512) NOT NULL,
-    publicPem VARCHAR(2048) NOT NULL,
-    _privatePemCipher VARCHAR(2048) NOT NULL,
-    _salt VARCHAR(512) NOT NULL,
+    password VARCHAR(1024) NOT NULL,
+    salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS tokens(
-    accessToken VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512),
+    accessToken VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
     expires VARCHAR(512) NOT NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appId VARCHAR(128) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
-    name VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS apps(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appStoreId VARCHAR(512) NOT NULL,
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appStoreId VARCHAR(128) NOT NULL,
     version VARCHAR(32),
     installationState VARCHAR(512) NOT NULL,
     installationProgress VARCHAR(512),
     runState VARCHAR(512),
     healthy INTEGER,
     containerId VARCHAR(128),
-    manifestJson VARCHAR,
+    manifestJson VARCHAR(2048),
     httpPort INTEGER,
-    location VARCHAR(512) NOT NULL UNIQUE,
+    location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
     accessRestriction VARCHAR(512),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
-    hostPort VARCHAR(5) NOT NULL UNIQUE,
+    hostPort INTEGER NOT NULL UNIQUE,
     containerPort VARCHAR(5) NOT NULL,
-    appId VARCHAR(512) NOT NULL,
+    appId VARCHAR(128) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
 CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    authCode VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128) NOT NULL,
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
-    key VARCHAR(512) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL UNIQUE,
     value VARCHAR(512),
-    PRIMARY KEY(key));
+    PRIMARY KEY(name));
+
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(128) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id));
 

migrations/schema.sql

@@ -1,74 +1,82 @@
 #### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 #### This file is not used by any code and is here to document the latest schema
 
+#### General ideas
+#### Default char set is utf8 and DEFAULT COLLATE is utf8_bin. Collate affects comparisons in WHERE and ORDER
+#### Strict mode is enabled
+#### VARCHAR - stored as part of table row (use for strings)
+#### TEXT - stored offline from table row (use for strings)
+#### BLOB - stored offline from table row (use for binary data)
+#### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
+
 CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(512) NOT NULL,
-    email VARCHAR(512) NOT NULL,
-    _password VARCHAR(512) NOT NULL,
-    publicPem VARCHAR(2048) NOT NULL,
-    _privatePemCipher VARCHAR(2048) NOT NULL,
-    _salt VARCHAR(512) NOT NULL,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    email VARCHAR(254) NOT NULL UNIQUE,
+    password VARCHAR(1024) NOT NULL,
+    salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS tokens(
-    accessToken VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512),
+    accessToken VARCHAR(128) NOT NULL UNIQUE,
+    identifier VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
-    expires VARCHAR(512) NOT NULL,
+    expires BIGINT NOT NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appId VARCHAR(128) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
-    name VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS apps(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appStoreId VARCHAR(512) NOT NULL,
-    version VARCHAR(32),
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appStoreId VARCHAR(128) NOT NULL,
     installationState VARCHAR(512) NOT NULL,
     installationProgress VARCHAR(512),
     runState VARCHAR(512),
-    healthy INTEGER,
+    health VARCHAR(128),
     containerId VARCHAR(128),
-    manifestJson VARCHAR,
-    httpPort INTEGER,
-    location VARCHAR(512) NOT NULL UNIQUE,
+    manifestJson VARCHAR(2048),
+    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
+    location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
     accessRestriction VARCHAR(512),
+    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    lastBackupId VARCHAR(128),
+    lastBackupConfigJson VARCHAR(2048), // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
-    hostPort VARCHAR(5) NOT NULL UNIQUE,
-    containerPort VARCHAR(5) NOT NULL,
-    appId VARCHAR(512) NOT NULL,
+    hostPort INTEGER NOT NULL UNIQUE,
+    environmentVariable VARCHAR(128) NOT NULL,
+    appId VARCHAR(128) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
-CREATE TABLE IF NOT EXISTS appAddonConfigs(
-    appId VARCHAR(512) NOT NULL,
-    addonId VARCHAR(32) NOT NULL,
-    value VARCHAR(512),
-    FOREIGN KEY(appId) REFERENCES apps(id));
-
 CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    authCode VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128) NOT NULL,
+    expiresAt BIGINT NOT NULL,
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
-    key VARCHAR(512) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL UNIQUE,
     value VARCHAR(512),
-    PRIMARY KEY(key));
+    PRIMARY KEY(name));
+
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(128) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id));
 

oauthproxy.js

@@ -6,7 +6,9 @@ require('supererror')({ splatchError: true });
 
 var express = require('express'),
     url = require('url'),
+    uuid = require('node-uuid'),
     async = require('async'),
+    superagent = require('superagent'),
     assert = require('assert'),
     debug = require('debug')('box:proxy'),
     proxy = require('proxy-middleware'),
@@ -14,9 +16,12 @@ var express = require('express'),
     database = require('./src/database.js'),
     appdb = require('./src/appdb.js'),
     clientdb = require('./src/clientdb.js'),
-    config = require('./config.js'),
+    config = require('./src/config.js'),
     http = require('http');
 
+// Allow self signed certs!
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+
 var gSessions = {};
 var gProxyMiddlewareCache = {};
 var gApp = express();
@@ -26,7 +31,7 @@ var CALLBACK_URI = '/callback';
 var PORT = 4000;
 
 function startServer(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     gHttpServer.on('error', console.error);
 
@@ -34,20 +39,80 @@ function startServer(callback) {
         keys: ['blue', 'cheese', 'is', 'something']
     }));
 
+    // ensure we have a in memory store for the session to cache client information
     gApp.use(function (req, res, next) {
-        if (req.session && gSessions[req.session.sessid]) return next();
+        assert.strictEqual(typeof req.session, 'object');
 
-        if (req.path === CALLBACK_URI) {
-            // FIXME we need to exchange the authCode and verify it
-            req.session.sessid = req.query.authCode;
+        if (!req.session.id || !gSessions[req.session.id]) {
+            req.session.id = uuid.v4();
+            gSessions[req.session.id] = {};
+        }
 
-            // this is a simple in memory auth store
-            gSessions[req.session.sessid] = 'ok';
+        // attach the session data to the requeset
+        req.sessionData = gSessions[req.session.id];
 
-            debug('user verified.');
+        next();
+    });
 
-            // now redirect to the actual initially requested URL
-            res.redirect(req.session.returnTo);
+    gApp.use(function verifySession(req, res, next) {
+        assert.strictEqual(typeof req.sessionData, 'object');
+
+        if (!req.sessionData.accessToken) {
+            req.authenticated = false;
+            return next();
+        }
+
+        superagent.get(config.adminOrigin() + '/api/v1/profile').query({ access_token: req.sessionData.accessToken}).end(function (error, result) {
+            if (error) {
+                console.error(error);
+                req.authenticated = false;
+            } else if (result.statusCode !== 200) {
+                req.sessionData.accessToken = null;
+                req.authenticated = false;
+            } else {
+                req.authenticated = true;
+            }
+
+            next();
+        });
+    });
+
+    gApp.use(function (req, res, next) {
+        // proceed if we are authenticated
+        if (req.authenticated) return next();
+
+        if (req.path === CALLBACK_URI && req.sessionData.returnTo) {
+            // exchange auth code for an access token
+            var query = {
+                response_type: 'token',
+                client_id: req.sessionData.clientId
+            };
+
+            var data = {
+                grant_type: 'authorization_code',
+                code: req.query.code,
+                redirect_uri: req.sessionData.returnTo,
+                client_id: req.sessionData.clientId,
+                client_secret: req.sessionData.clientSecret
+            };
+
+            superagent.post(config.adminOrigin() + '/api/v1/oauth/token').query(query).send(data).end(function (error, result) {
+                if (error) {
+                    console.error(error);
+                    return res.send(500, 'Unable to contact the oauth server.');
+                }
+                if (result.statusCode !== 200) {
+                    console.error('Failed to exchange auth code for a token.', result.statusCode, result.body);
+                    return res.send(500, 'Failed to exchange auth code for a token.');
+                }
+
+                req.sessionData.accessToken = result.body.access_token;
+
+                debug('user verified.');
+
+                // now redirect to the actual initially requested URL
+                res.redirect(req.sessionData.returnTo);
+            });
         } else {
             var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
 
@@ -70,13 +135,14 @@ function startServer(callback) {
                         return res.send(500, 'Unknown OAuth client.');
                     }
 
-                    req.session.port = port;
-                    req.session.returnTo =  result.redirectURI + req.path;
+                    req.sessionData.port = port;
+                    req.sessionData.returnTo = result.redirectURI + req.path;
+                    req.sessionData.clientId = result.id;
+                    req.sessionData.clientSecret = result.clientSecret;
 
-                    var callbackURL = result.redirectURI + CALLBACK_URI;
+                    var callbackUrl = result.redirectURI + CALLBACK_URI;
                     var scope = 'profile,roleUser';
-                    var clientId = result.clientId;
-                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + clientId + '&redirect_uri=' + callbackURL + '&scope=' + scope;
+                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + result.id + '&redirect_uri=' + callbackUrl + '&scope=' + scope;
 
                     debug('begin OAuth flow for client %s.', result.name);
 
@@ -88,7 +154,7 @@ function startServer(callback) {
     });
 
     gApp.use(function (req, res, next) {
-        var port = req.session.port;
+        var port = req.sessionData.port;
 
         debug('proxy request for port %s with path %s.', port, req.path);
 

package.json

@@ -1,96 +1,100 @@
 {
-  "name": "yellowtent",
-  "description": "Yellow tent",
+  "name": "Cloudron",
+  "description": "Main code for a cloudron",
   "version": "0.0.1",
   "private": "true",
   "author": {
-    "name": "Yellow tent authors",
-    "email": "girish@forwardbias.in"
+    "name": "Cloudron authors"
   },
   "repository": {
     "type": "git"
   },
   "engines": [
-    "node >= 0.10.0"
+    "node >= 0.12.0"
   ],
   "bin": {
-    "yellowtent": "./server.js"
+    "cloudron": "./app.js"
   },
   "dependencies": {
-    "async": "^0.6.2",
-    "body-parser": "~1.9.3",
-    "commander": "^2.2.0",
+    "async": "^1.2.1",
+    "body-parser": "^1.13.1",
+    "cloudron-manifestformat": "^1.6.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.8",
-    "connect-timeout": "~1.4.0",
-    "cookie-parser": "1.1.0",
+    "connect-lastmile": "0.0.13",
+    "connect-timeout": "^1.5.0",
+    "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
-    "csurf": "^1.6.1",
-    "db-migrate": "~0.7.1",
-    "debug": "~0.8.1",
-    "dockerode": "~2.0.5",
-    "ejs": "^1.0.0",
-    "encfs": "^0.1.1",
-    "express": "~4.2.0",
-    "express-session": "~1.1.0",
-    "js-yaml": "~3.2.2",
-    "json": "~9.0.1",
-    "memorystream": "~0.2.0",
-    "mime": "^1.2.11",
-    "mkdirp": "~0.3.5",
-    "morgan": "~1.0.1",
-    "multiparty": "http://registry.npmjs.org/multiparty/-/multiparty-4.0.0.tgz",
-    "native-dns": "~0.6.1",
-    "node-uuid": "^1.4.1",
-    "nodejs-disks": "~0.2.1",
-    "nodemailer": "~1.3.0",
-    "nodemailer-smtp-transport": "~0.1.13",
+    "cron": "^1.0.9",
+    "csurf": "^1.6.6",
+    "db-migrate": "^0.9.2",
+    "debug": "^2.2.0",
+    "dockerode": "^2.2.2",
+    "ejs": "^2.2.4",
+    "ejs-cli": "^1.0.1",
+    "express": "^4.12.4",
+    "express-session": "^1.11.3",
+    "hat": "0.0.3",
+    "json": "^9.0.3",
+    "ldapjs": "^0.7.1",
+    "memorystream": "^0.3.0",
+    "mime": "^1.3.4",
+    "morgan": "^1.6.0",
+    "multiparty": "^4.1.2",
+    "mysql": "^2.7.0",
+    "native-dns": "^0.7.0",
+    "node-uuid": "^1.4.3",
+    "nodejs-disks": "^0.2.1",
+    "nodemailer": "^1.3.0",
+    "nodemailer-smtp-transport": "^1.0.3",
     "oauth2orize": "^1.0.1",
-    "once": "^1.3.0",
-    "passport": "~0.2.1",
+    "once": "^1.3.2",
+    "passport": "^0.2.2",
     "passport-http": "^0.2.2",
     "passport-http-bearer": "^1.0.1",
     "passport-local": "^1.0.0",
-    "passport-oauth2-client-password": "~0.1.2",
-    "password-generator": "~0.2.3",
-    "proxy-middleware": "~0.5.1",
-    "readdirp": "^1.0.1",
-    "rimraf": "^2.2.6",
-    "safetydance": "0.0.12",
-    "semver": "~4.2.0",
-    "serve-favicon": "~2.1.7",
-    "split": "^0.3.0",
-    "sqlite3": "^3.0.0",
-    "superagent": "~0.17.0",
-    "supererror": "~0.6.0",
-    "underscore": "~1.7.0",
-    "ursa": "^0.8.0",
-    "validator": "~3.22.1"
+    "passport-oauth2-client-password": "^0.1.2",
+    "password-generator": "^1.0.0",
+    "proxy-middleware": "^0.13.0",
+    "safetydance": "0.0.19",
+    "semver": "^4.3.6",
+    "serve-favicon": "^2.2.0",
+    "split": "^1.0.0",
+    "superagent": "~0.21.0",
+    "supererror": "^0.7.0",
+    "supervisord-eventlistener": "^0.1.0",
+    "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "underscore": "^1.7.0",
+    "valid-url": "^1.0.9",
+    "validator": "^3.30.0"
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "~2.0.23",
+    "aws-sdk": "^2.1.10",
+    "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",
-    "gulp": "^3.8.10",
+    "gulp": "^3.8.11",
+    "gulp-autoprefixer": "^2.3.0",
     "gulp-concat": "^2.4.3",
     "gulp-ejs": "^1.0.0",
-    "gulp-sourcemaps": "^1.3.0",
-    "hock": "~0.2.5",
-    "husky": "~0.6.2",
+    "gulp-minify-css": "^1.1.3",
+    "gulp-sass": "^2.0.1",
+    "gulp-serve": "^1.0.0",
+    "gulp-sourcemaps": "^1.5.2",
+    "gulp-uglify": "^1.1.0",
+    "hock": "~1.2.0",
     "istanbul": "*",
     "mocha": "*",
-    "nock": "~0.43.1",
-    "redis": "~0.12.1",
-    "s3-cli": "~0.11.1",
-    "semver": "~4.2.0",
-    "sinon": "~1.10.3"
+    "nock": "^2.6.0",
+    "node-sass": "^3.0.0-alpha.0",
+    "redis": "^0.12.1",
+    "sinon": "^1.12.2",
+    "yargs": "^3.15.0"
   },
   "scripts": {
-    "create_testdb": "rm -rf $HOME/.yellowtenttest/*; mkdir -p $HOME/.yellowtenttest/data; NODE_ENV=test DATABASE_URL=sqlite3:///$HOME/.yellowtenttest/data/cloudron.sqlite node_modules/.bin/db-migrate up",
-    "migrate": "mkdir -p $HOME/.yellowtent/data; DATABASE_URL=sqlite3:///$HOME/.yellowtent/data/cloudron.sqlite node_modules/.bin/db-migrate up",
-    "migrate_data": "DATABASE_URL=sqlite3:///home/yellowtent/data/cloudron.sqlite db-migrate up",
-    "test": "scripts/checkInstall && npm run-script create_testdb && NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
+    "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
+    "migrate_test": "BOX_ENV=test DATABASE_URL=mysql://root:@localhost/boxtest node_modules/.bin/db-migrate up",
+    "test": "npm run migrate_test && src/test/setupTest && BOX_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
     "postmerge": "/bin/true",
     "precommit": "/bin/true",
     "prepush": "npm test",

release/publish.sh

@@ -1,144 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-readonly JSON="${SOURCE_DIR}/node_modules/.bin/json"
-[ "$(uname -s)" == "Darwin" ] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-readonly VERSIONS_URL_DEV="https://s3.amazonaws.com/cloudron-releases/versions-dev.json"
-readonly VERSIONS_S3_URL_DEV="s3://cloudron-releases/versions-dev.json"
-readonly VERSIONS_URL_STAGING="https://s3.amazonaws.com/cloudron-releases/versions-staging.json"
-readonly VERSIONS_S3_URL_STAGING="s3://cloudron-releases/versions-staging.json"
-
-if [[ ! -f "${SOURCE_DIR}/../installer/scripts/digitalOceanFunctions.sh" ]]; then
-    echo "Could not locate digitalOceanFunctions.sh"
-    exit 1
-fi
-
-source "${SOURCE_DIR}/../installer/scripts/digitalOceanFunctions.sh"
-
-new_versions_file=""
-source_tarball_url=""
-image_id=""
-cmd=""
-new_version=""
-changelog="If I told you, I'd have to kill you"
-upgrade="autodetect"
-versions_url="${VERSIONS_URL_DEV}"
-versions_s3_url="${VERSIONS_S3_URL_DEV}"
-
-args=$($GNU_GETOPT -o "" -l "dev,staging,code:,image:,rerelease,new,list,revert,changelog:,release:,upgrade" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --dev) shift;;
-    --staging) versions_url="${VERSIONS_URL_STAGING}"; versions_s3_url="${VERSIONS_S3_URL_STAGING}"; shift;;
-    --code) source_tarball_url="$2"; shift 2;;
-    --image) image_id="$2"; shift 2;;
-    --rerelease) cmd="rerelease"; shift;;
-    --new) cmd="new"; shift;;
-    --release) cmd="release"; new_versions_file="$2"; shift 2;;
-    --list) cmd="list"; shift;;
-    --revert) cmd="revert"; shift;;
-    --changelog) changelog="$2"; shift 2;;
-    --upgrade) upgrade="true"; shift;;
-    --) shift; break;;
-    *) echo "Unknown option $2"; exit;;
-    esac
-done
-
-shift $(expr $OPTIND - 1)
-
-download_current() {
-    versions_url="$1"
-    # download the existing version file if the user hasn't provided one
-    local current_versions_file=$(mktemp -t box-versions 2>/dev/null || mktemp)
-
-    if ! wget -q -O "${current_versions_file}" "${versions_url}"; then
-        echo "Error downloading versions file"
-        exit 1
-    fi
-
-    echo "${current_versions_file}"
-}
-
-if [[ "${cmd}" == "list" ]]; then
-    cat "$(download_current "${versions_url}")"
-    exit 0
-elif [[ "${cmd}" == "release" ]]; then
-    if [[ ! -f "${new_versions_file}" ]]; then
-        echo "${new_versions_file} cannot be found"
-        exit 1
-    fi
-elif [[ "${cmd}" == "new" ]]; then
-    if [[ -z "${source_tarball_url}" || -z "${image_id}" ]]; then
-        echo "--code and --image is required"
-        exit 1
-    fi
-
-    new_version="0.0.1"
-    image_name=$(get_image_name "${image_id}")
-
-    new_versions_file=$(mktemp -t box-versions 2>/dev/null || mktemp)
-    cat > "${new_versions_file}" <<EOF
-    {
-        "0.0.1": {
-            "sourceTarballUrl": "${source_tarball_url}",
-            "imageId": ${image_id},
-            "imageName": "${image_name}",
-            "changelog": [ "Let's start at the very beginning, a very good way to start" ],
-            "date": "$(date -u)",
-            "next": null
-        }
-    }
-EOF
-elif [[ "${cmd}" == "revert" ]]; then
-    new_versions_file=$(download_current "${versions_url}")
-    last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 1)
-    second_last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 2 | head -n 1)
-
-    echo "Removing $last_version and making $second_last_version the last release"
-    $JSON -q -I -f "${new_versions_file}" -e "delete this['${last_version}']"
-    $JSON -q -I -f "${new_versions_file}" -e "this['${second_last_version}'].next = null"
-else
-    new_versions_file=$(download_current "${versions_url}")
-    # modify existing versions.json
-    if [[ -z "${source_tarball_url}" && -z "${image_id}" && "${cmd}" != "rerelease" ]]; then
-        echo "--code or --image is required"
-        exit 1
-    fi
-
-    readonly last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 1)
-    if [[ -z "${source_tarball_url}" ]]; then
-        source_tarball_url=$($JSON -f "${new_versions_file}" -D, "${last_version},sourceTarballUrl")
-        echo "Using the previous code url : ${source_tarball_url}"
-    fi
-    if [[ -z "${image_id}" ]]; then
-        image_id=$($JSON -f "${new_versions_file}" -D, "${last_version},imageId")
-        echo "Using the previous image id : ${image_id}"
-    fi
-
-    if [[ "${upgrade}" == "autodetect" ]]; then
-        old_image_id=$($JSON -f "${new_versions_file}" -D, "${last_version},imageId")
-        upgrade=$([[ "${old_image_id}" != "${image_id}" ]] && echo "true" || echo "false")
-    fi
-
-    new_version=$($SOURCE_DIR/node_modules/.bin/semver -i "${last_version}")
-    echo "Releasing version ${new_version}"
-    image_name=$(get_image_name "${image_id}")
-
-    $JSON -q -I -f "${new_versions_file}" -e "this['${last_version}'].next = '${new_version}'"
-    $JSON -q -I -f "${new_versions_file}" -e "this['${new_version}'] = { 'sourceTarballUrl': '${source_tarball_url}', 'imageId': ${image_id}, 'imageName': '${image_name}', 'changelog': [ \"${changelog}\" ], 'upgrade': ${upgrade}, 'date': '$(date -u)', 'next': null }"
-fi
-
-echo "Verifying new versions file"
-$SOURCE_DIR/release/verify.js "${new_versions_file}"
-
-echo "Uploading new versions file"
-$SOURCE_DIR/node_modules/.bin/s3-cli put --acl-public --default-mime-type "application/json" "${new_versions_file}" "${versions_s3_url}"
-
-cat "${new_versions_file}"
-

release/stage.sh

@@ -1,102 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-readonly JSON="${SOURCE_DIR}/node_modules/.bin/json"
-readonly SEMVER="${SOURCE_DIR}/node_modules/.bin/semver"
-[ $(uname -s) == "Darwin" ] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-readonly VERSIONS_URL_DEV="https://s3.amazonaws.com/cloudron-releases/versions-dev.json"
-readonly VERSIONS_URL_STAGING="https://s3.amazonaws.com/cloudron-releases/versions-staging.json"
-readonly VERSIONS_S3_URL_STAGING="s3://cloudron-releases/versions-staging.json"
-
-verify_tag() {
-    tag="$1"
-    git rev-parse --verify "tags/$1" 2>/dev/null
-}
-
-download() {
-    # download the existing version file if the user hasn't provided one
-    local tmp_file=$(mktemp -t stage 2>/dev/null || mktemp)
-
-    if wget -q -O "${tmp_file}" "${1}"; then
-        echo "${tmp_file}"
-    fi
-}
-
-read_changelog() {
-    version="$1"
-    changelog_file="${SOURCE_DIR}/release/changelogs/changelog-$1"
-
-    if [[ -f "${changelog_file}" ]]; then
-        cat "${changelog_file}" | grep -v "^#"
-    fi
-}
-
-if [[ $# -lt 1 ]]; then
-    echo "Usage: stage.sh <dev-version>"
-    exit 1
-fi
-
-dev_version="$1"
-
-dev_versions_file=$(download "${VERSIONS_URL_DEV}")
-if [[ -z "${dev_versions_file}" ]]; then
-    echo "Error downloading dev versions file"
-    exit 1
-fi
-
-dev_version_info=$($JSON -f "${dev_versions_file}" -D, "${dev_version}")
-if [[ -z "${dev_version_info}" ]]; then
-    echo "No such version in dev ${dev_versions_file} ${dev_version}"
-    exit 1
-fi
-
-staging_versions_file=$(download "${VERSIONS_URL_STAGING}") ## TODO: this can fail
-if [[ -z "${staging_versions_file}" ]]; then
-    echo "Creating new staging release file"
-    staging_versions_file=$(mktemp -t stage 2>/dev/null || mktemp)
-    echo "{}" > ${staging_versions_file}
-    readonly staging_last_version="0.0.0"
-    staging_new_version="0.0.1"
-    upgrade="false"
-else
-    readonly staging_last_version=$(cat "${staging_versions_file}" | $JSON -ka | tail -n 1)
-    staging_new_version=$($SEMVER -i "${staging_last_version}")
-    $JSON -q -I -f "${staging_versions_file}" -e "this['${staging_last_version}'].next = '${staging_new_version}'"
-
-    last_image_id=$($JSON -f "${staging_versions_file}" -D, "${staging_last_version},imageId")
-    new_image_id=$($JSON -f "${dev_versions_file}" -D, "${dev_version},imageId")
-
-    upgrade=$([[ "${last_image_id}" != "${new_image_id}" ]] && echo "true" || echo "false")
-fi
-
-#TODO: check if the tag matches the sha1 in the sourceTarballUrl
-if ! verify_tag "v${staging_new_version}"; then
-    echo "No git tag named v${staging_new_version} found"
-    exit 1
-fi
-
-changelog=$(read_changelog "${staging_new_version}")
-if [[ -z "${changelog}" ]]; then
-    echo "Missing changelog file or empty change log"
-    exit 1
-fi
-
-echo "Releasing version ${staging_new_version}"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'] = ${dev_version_info}"
-#$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].changelog = '[ "${changelog}" ]'"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].upgrade = ${upgrade}"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].date = '$(date -u)'"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].next = null"
-
-echo "Verifying new versions file"
-$SOURCE_DIR/release/verify.js "${staging_versions_file}"
-
-echo "Uploading new versions file"
-$SOURCE_DIR/node_modules/.bin/s3-cli put --acl-public --default-mime-type "application/json" "${staging_versions_file}" "${VERSIONS_S3_URL_STAGING}"
-
-cat "${staging_versions_file}" | tee $SOURCE_DIR/release/versions-staging.json
-

release/verify.js

@@ -1,55 +0,0 @@
-#!/usr/bin/env node
-
-var AWS = require('aws-sdk'),
-    fs = require('fs'),
-    path = require('path'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    url = require('url');
-
-function die(msg) {
-    console.error(msg);
-    process.exit(1);
-}
-
-function verify(versionsFileName) {
-    // check if the json is valid
-    var versionsJson = safe.JSON.parse(fs.readFileSync(versionsFileName));
-    if (!versionsJson) {
-        die(versionsFileName + ' is not valid json : ' + safe.error);
-    }
-
-    // check all the keys
-    var sortedVersions = Object.keys(versionsJson).sort();
-    sortedVersions.forEach(function (version, index) {
-        if (typeof versionsJson[version].imageId !== 'number') die('version ' + version + ' does not have proper imageId');
-        if (typeof versionsJson[version].imageName !== 'string' || !versionsJson[version].imageName.length) die('version ' + version + ' does not have proper imageName');
-        if ('changeLog' in versionsJson[version] && !util.isArray(versionsJson[version].changeLog)) die('version ' + version + ' does not have proper changeLog');
-        if (typeof versionsJson[version].date !== 'string' || ((new Date(versionsJson[version].date)).toString() === 'Invalid Date')) die('invalid date or missing date');
-        if (versionsJson[version].next !== null && typeof versionsJson[version].next !== 'string') die('version ' + version + ' does not have proper next');
-        if (typeof versionsJson[version].sourceTarballUrl !== 'string') die('version ' + version + ' does not have proper sourceTarballUrl');
-        var tarballUrl = url.parse(versionsJson[version].sourceTarballUrl);
-        if (tarballUrl.protocol !== 'https:') die('sourceTarballUrl must be https');
-        if (!/.tar.gz$/.test(tarballUrl.path)) die('sourceTarballUrl must be tar.gz');
-
-        var nextVersion = versionsJson[version].next;
-        // despite having the 'next' field, the appstore code currently relies on all versions being sorted based on semver.compare (see boxversions.js)
-        if (nextVersion && semver.gt(version, nextVersion)) die('next version cannot be less than current @' + version);
-    });
-
-    // check that package.json version is in versions.json
-    var currentVersion = require('../package.json').version;
-    if (sortedVersions.indexOf(currentVersion) === -1) {
-        die('package.json version is not present in versions.json');
-    }
-}
-
-if (process.argv.length === 3) {
-    verify(process.argv[2]);
-    process.exit(0);
-} else {
-    console.log('verify.js <versions_file>');
-}
-
-
-

release/versions-stable.json

@@ -1,7 +0,0 @@
-{
-    "0.0.1": {
-        "revision": "9f09f8e7a8633e8b3341bb9c610f5f631ccd288c",
-        "imageId": 7531071,
-        "next": null
-    }
-}

run.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-echo
-echo "Starting Cloudron at port 443"
-echo
-
-readonly BOX_SRC_DIR="$(cd $(dirname "$0"); pwd)"
-readonly NGINX_ROOT=~/.yellowtent/nginx
-readonly PROVISION_VERSION=0.1
-readonly PROVISION_BOX_VERSIONS_URL=0.1
-readonly DATA_DIR=~/.yellowtent/data
-readonly FQDN=admin-localhost
-
-mkdir -p "${NGINX_ROOT}/applications"
-mkdir -p "${NGINX_ROOT}/cert"
-mkdir -p "${DATA_DIR}"
-
-# get the database current
-npm run-script migrate
-
-cp setup/start/nginx/nginx.conf "${NGINX_ROOT}/nginx.conf"
-cp setup/start/nginx/mime.types "${NGINX_ROOT}/mime.types"
-cp setup/start/nginx/cert/* "${NGINX_ROOT}/cert/"
-
-# adjust the generated nginx config for local use
-touch "${NGINX_ROOT}/naked_domain.conf"
-sed -e "s/##ADMIN_FQDN##/${FQDN}/" -e "s|##BOX_SRC_DIR##|${BOX_SRC_DIR}|" setup/start/nginx/admin.conf_template > "${NGINX_ROOT}/applications/admin.conf"
-sed -e "s/user www-data/user ${USER}/" -i "${NGINX_ROOT}/nginx.conf"
-
-# add webadmin oauth client
-readonly WEBADMIN_ID=abcdefg
-readonly WEBADMIN_SCOPES="root,profile,users,apps,settings,roleAdmin"
-sqlite3 "${DATA_DIR}/cloudron.sqlite" "INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES (\"${WEBADMIN_ID}\", \"webadmin\", \"cid-webadmin\", \"secret-webadmin\", \"WebAdmin\", \"https://${FQDN}\", \"${WEBADMIN_SCOPES}\")"
-
-# start nginx
-sudo nginx -c nginx.conf -p "${NGINX_ROOT}"
-

scripts/checkInstall

@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-
-# reset sudo timestamp to avoid wrong success
-sudo -k || sudo --reset-timestamp
-
-# checks if all scripts are sudo access
-scripts=("${SOURCE_DIR}/src/scripts/rmappdir.sh" \
-         "${SOURCE_DIR}/src/scripts/reloadnginx.sh" \
-         "${SOURCE_DIR}/src/scripts/backup.sh" \
-         "${SOURCE_DIR}/src/scripts/reboot.sh" \
-         "${SOURCE_DIR}/src/scripts/reloadcollectd.sh")
-
-for script in "${scripts[@]}"; do
-    if [[ $(sudo -n "${script}" --check 2>/dev/null) != "OK" ]]; then
-        echo ""
-        echo "${script} does not have sudo access."
-        echo "You have to add the lines below to /etc/sudoers.d/yellowtent."
-        echo ""
-        echo "Defaults!${script} env_keep=HOME"
-        echo "${USER} ALL=(ALL) NOPASSWD: ${script}"
-        echo ""
-        exit 1
-    fi
-done
-
-if ! docker inspect girish/test:0.6 >/dev/null 2>/dev/null; then
-    echo "docker pull girish/test:0.6 for tests to run"
-    exit 1
-fi
-
-if ! docker inspect girish/redis:0.1 >/dev/null 2>/dev/null; then
-    echo "docker pull girish/redis:0.1 for tests to run"
-    exit 1
-fi
-
-exit 0

scripts/createSourceTarball

@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-[[ ! -f "${HOME}/.s3cfg" ]] && echo "~/.s3cfg missing" && exit 1
-
-# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
-# brew install gnu-getopt to get the GNU getopt on OS X
-[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-args=$(${GNU_GETOPT} -o "" -l "revision:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-commitish="HEAD"
-
-while true; do
-    case "$1" in
-    --revision) commitish="$2"; shift 2;;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-done
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint?
-
-version=$(cd "${SOURCE_DIR}" && git rev-parse "${commitish}")
-bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
-bundle_file="${TMPDIR}/box-${version}.tar.gz"
-
-chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
-echo "Checking out code [${version}] into ${bundle_dir}"
-(cd "${SOURCE_DIR}" && git archive --format=tar HEAD | (cd "${bundle_dir}" && tar xf -))
-
-echo "Installing modules"
-cd "${bundle_dir}" && npm install --production
-
-cd "${bundle_dir}" && tar czvf "${bundle_file}" .
-
-echo "Uploading bundle to S3"
-${SOURCE_DIR}/node_modules/.bin/s3-cli put --acl-public "${bundle_file}" "s3://cloudron-releases/box-${version}.tar.gz"
-
-echo "Cleaning up ${bundle_dir}"
-rm -rf "${bundle_dir}" "${bundle_file}"
-

setup/DESIGN.md

@@ -0,0 +1,57 @@
+This document gives the design of this setup code.
+
+box code should be delivered in the form of a (docker) container.
+This is not the case currently but we want to do structure the code
+in spirit that way.
+
+### container.sh
+This contains code that essential goes into Dockerfile.
+
+This file contains static configuration over a base image. Currently,
+the yellowtent user is created in the installer base image but it
+could very well be placed here.
+
+The idea is that the installer would simply remove the old box container
+and replace it with a new one for an update.
+
+Because we do not package things as Docker yet, we should be careful
+about the code here. We have to expect remains of an older setup code.
+For example, older supervisor or nginx configs might be around.
+
+The config directory is _part_ of the container and is not a VOLUME.
+Which is to say that the files will be nuked from one update to the next.
+
+The data directory is a VOLUME. Contents of this directory are expected
+to survive an update. This is a good place to place config files that
+are "dynamic" and need to survive restarts. For example, the infra
+version (see below) or the mysql/postgresql data etc.
+
+### start.sh
+  * It is called in 3 modes - new, update, restore.
+
+  * The first thing this does is to do the static container.sh setup.
+
+  * It then downloads any box restore data and restores the box db from the
+    backup.
+
+  * It then proceeds to call the db-migrate script.
+
+  * It then does dynamic configuration like setting up nginx, collectd.
+
+  * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
+
+  * supervisor is then started
+
+setup_infra.sh
+This setups containers like graphite, mail and the addons containers.
+
+Containers are relaunched based on the INFRA_VERSION. The script compares
+the version here with the version in the file DATA_DIR/INFRA_VERSION.
+
+If they match, the containers are not recreated and nothing is to be done.
+nginx, collectd configs are part of data already and containers are running.
+
+If they do not match, it deletes all containers (including app containers) and starts
+them all afresh. Important thing here is that, DATA_DIR is never removed across
+updates. So, it is only the containers being recreated and not the data.
+

setup/INFRA_VERSION

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# If you change the infra version, be sure to put a warning
+# in the change log
+
+INFRA_VERSION=4

setup/argparser.sh

@@ -3,34 +3,60 @@
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 json="${script_dir}/../node_modules/.bin/json"
 
+# IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
+arg_api_server_origin=""
 arg_box_versions_url=""
+arg_fqdn=""
+arg_is_custom_domain="false"
+arg_restore_key=""
+arg_restore_url=""
+arg_retire="false"
 arg_tls_cert=""
 arg_tls_key=""
-arg_app_server_url=""
-arg_fqdn=""
 arg_token=""
 arg_version=""
-arg_is_custom_domain="false"
+arg_web_server_origin=""
 
-args=$(getopt -o "" -l "boxversionsurl:,data:,version:" -n "$0" -- "$@")
+args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
-    --boxversionsurl) arg_box_versions_url="$2";;
+    --retire)
+        arg_retire="true"
+        shift
+        ;;
     --data)
-        read -r arg_app_server_url arg_fqdn arg_token arg_is_custom_domain <<EOF
-        $(echo "$2" | $json appServerUrl fqdn token isCustomDomain | tr '\n' ' ')
+        # only read mandatory non-empty parameters here
+        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_token arg_is_custom_domain arg_box_versions_url arg_version <<EOF
+        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn token isCustomDomain boxVersionsUrl version | tr '\n' ' ')
 EOF
+        # read possibly empty parameters here
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
+
+        arg_restore_url=$(echo "$2" | $json restoreUrl)
+        [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
+
+        arg_restore_key=$(echo "$2" | $json restoreKey)
+        [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
+
+        shift 2
         ;;
-    --version) arg_version="$2";;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
     esac
-
-    shift 2
 done
 
-
+echo "Parsed arguments:"
+echo "api server: ${arg_api_server_origin}"
+echo "box versions url: ${arg_box_versions_url}"
+echo "fqdn: ${arg_fqdn}"
+echo "custom domain: ${arg_is_custom_domain}"
+echo "restore key: ${arg_restore_key}"
+echo "restore url: ${arg_restore_url}"
+echo "tls cert: ${arg_tls_cert}"
+echo "tls key: ${arg_tls_key}"
+echo "token: ${arg_token}"
+echo "version: ${arg_version}"
+echo "web server: ${arg_web_server_origin}"

setup/container.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# This file can be used in Dockerfile
+
+readonly container_files="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/container"
+
+readonly CONFIG_DIR="/home/yellowtent/configs"
+readonly DATA_DIR="/home/yellowtent/data"
+
+########## create config directory
+rm -rf "${CONFIG_DIR}"
+sudo -u yellowtent mkdir "${CONFIG_DIR}"
+
+########## logrotate (default ubuntu runs this daily)
+rm -rf /etc/logrotate.d/*
+cp -r "${container_files}/logrotate/." /etc/logrotate.d/
+
+########## supervisor
+rm -rf /etc/supervisor/*
+cp -r "${container_files}/supervisor/." /etc/supervisor/
+
+########## sudoers
+rm /etc/sudoers.d/*
+cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
+
+########## collectd
+rm -rf /etc/collectd
+ln -sfF "${DATA_DIR}/collectd" /etc/collectd
+
+########## nginx
+# link nginx config to system config
+unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
+ln -s "${DATA_DIR}/nginx" /etc/nginx
+
+########## Enable services
+update-rc.d -f collectd defaults
+

setup/container/logrotate/cloudron

@@ -0,0 +1,6 @@
+/var/log/cloudron/*log {
+    missingok
+    notifempty
+    size 100k
+    nocompress
+}

setup/container/logrotate/supervisor

@@ -0,0 +1,7 @@
+/var/log/supervisor/*log {
+    missingok
+    copytruncate
+    notifempty
+    size 100k
+    nocompress
+}

setup/container/sudoers

@@ -0,0 +1,29 @@
+Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupbox.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restoreapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
+
+Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

setup/container/supervisor/conf.d/apphealthtask.conf

@@ -0,0 +1,10 @@
+[program:apphealthtask]
+command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/apphealthtask.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/box.conf

@@ -0,0 +1,10 @@
+[program:box]
+command=/usr/bin/node "/home/yellowtent/box/app.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/box.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/crashnotifier.conf

@@ -0,0 +1,11 @@
+[eventlistener:crashnotifier]
+command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
+events=PROCESS_STATE
+autostart=true
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/log/supervisor/crashnotifier.log
+stderr_logfile_maxbytes=50MB
+stderr_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/janitor.conf

@@ -0,0 +1,10 @@
+[program:janitor]
+command=/usr/bin/node "/home/yellowtent/box/janitor.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/janitor.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/oauthproxy.conf

@@ -0,0 +1,10 @@
+[program:oauthproxy]
+command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/oauthproxy.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/splash/nginx/mime.types

@@ -1,80 +0,0 @@
-
-types {
-    text/html                             html htm shtml;
-    text/css                              css;
-    text/xml                              xml;
-    image/gif                             gif;
-    image/jpeg                            jpeg jpg;
-    application/x-javascript              js;
-    application/atom+xml                  atom;
-    application/rss+xml                   rss;
-
-    text/mathml                           mml;
-    text/plain                            txt;
-    text/vnd.sun.j2me.app-descriptor      jad;
-    text/vnd.wap.wml                      wml;
-    text/x-component                      htc;
-
-    image/png                             png;
-    image/tiff                            tif tiff;
-    image/vnd.wap.wbmp                    wbmp;
-    image/x-icon                          ico;
-    image/x-jng                           jng;
-    image/x-ms-bmp                        bmp;
-    image/svg+xml                         svg svgz;
-    image/webp                            webp;
-
-    application/java-archive              jar war ear;
-    application/mac-binhex40              hqx;
-    application/msword                    doc;
-    application/pdf                       pdf;
-    application/postscript                ps eps ai;
-    application/rtf                       rtf;
-    application/vnd.ms-excel              xls;
-    application/vnd.ms-powerpoint         ppt;
-    application/vnd.wap.wmlc              wmlc;
-    application/vnd.google-earth.kml+xml  kml;
-    application/vnd.google-earth.kmz      kmz;
-    application/x-7z-compressed           7z;
-    application/x-cocoa                   cco;
-    application/x-java-archive-diff       jardiff;
-    application/x-java-jnlp-file          jnlp;
-    application/x-makeself                run;
-    application/x-perl                    pl pm;
-    application/x-pilot                   prc pdb;
-    application/x-rar-compressed          rar;
-    application/x-redhat-package-manager  rpm;
-    application/x-sea                     sea;
-    application/x-shockwave-flash         swf;
-    application/x-stuffit                 sit;
-    application/x-tcl                     tcl tk;
-    application/x-x509-ca-cert            der pem crt;
-    application/x-xpinstall               xpi;
-    application/xhtml+xml                 xhtml;
-    application/zip                       zip;
-
-    application/octet-stream              bin exe dll;
-    application/octet-stream              deb;
-    application/octet-stream              dmg;
-    application/octet-stream              eot;
-    application/octet-stream              iso img;
-    application/octet-stream              msi msp msm;
-
-    audio/midi                            mid midi kar;
-    audio/mpeg                            mp3;
-    audio/ogg                             ogg;
-    audio/x-m4a                           m4a;
-    audio/x-realaudio                     ra;
-
-    video/3gpp                            3gpp 3gp;
-    video/mp4                             mp4;
-    video/mpeg                            mpeg mpg;
-    video/quicktime                       mov;
-    video/webm                            webm;
-    video/x-flv                           flv;
-    video/x-m4v                           m4v;
-    video/x-mng                           mng;
-    video/x-ms-asf                        asx asf;
-    video/x-ms-wmv                        wmv;
-    video/x-msvideo                       avi;
-}

setup/splash/nginx/nginx.conf

@@ -1,63 +0,0 @@
-user www-data;
-worker_processes 1;
-pid /run/nginx.pid;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include mime.types;
-    default_type application/octet-stream;
-
-    access_log /var/log/nginx/access.log;
-    error_log /var/log/nginx/error.log;
-
-    root ##SETUP_WEBSITE_DIR##;
-    index index.html;
-
-    server {
-        listen 80;
-
-        location / {
-            # redirect everything to HTTPS
-            return 301 https://$host$request_uri;
-        }
-    }
-
-    # HTTPS server
-    server {
-        listen 443;
-
-        error_page 503 /index.html;
-
-        location /index.html {
-            # allow access to this page
-            add_header Cache-Control no-cache;
-        }
-
-        location /3rdparty/bootstrap.min.css {
-            # allow access to this page
-            add_header Cache-Control no-cache;
-        }
-
-        location /progress.json {
-            # allow access to this page
-            add_header Cache-Control no-cache;
-        }
-
-        location / {
-            return 503;
-        }
-
-        ssl on;
-        ssl_certificate      cert/host.cert;
-        ssl_certificate_key  cert/host.key;
-
-        ssl_session_timeout 5m;
-
-        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-        ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
-        ssl_prefer_server_ciphers on;
-    }
-}

setup/splash/website/index.html

@@ -1,36 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
-
-    <title> Cloudron Webadmin </title>
-
-    <link href="3rdparty/bootstrap.min.css" rel="stylesheet">
-</head>
-
-<body>
-    <!-- Modal update progress -->
-    <div class="modal show" id="updateProgressModal" tabindex="-1" role="dialog" aria-labelledby="updateProgressModalLabel" aria-hidden="true" data-keyboard    ="false" data-backdrop="static">
-        <div class="modal-dialog">
-            <div class="modal-content">
-                <div class="modal-header">
-                    <h4 class="modal-title" id="updateProgressModalLabel">Update in progress...</h4>
-                </div>
-                <div class="modal-body">
-                    <div class="progress progress-striped active">
-                        <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%"></div>
-                    </div>
-                </div>
-            </div>
-            <!-- /.modal-content -->
-        </div>
-        <!-- /.modal-dialog -->
-    </div>
-
-    <script>
-        setTimeout(location.reload.bind(location, true /* forceGet from server */), 10000);
-    </script>
-</body>
-
-</html>

setup/splashpage.sh

@@ -1,32 +1,40 @@
 #!/bin/bash
 
-set -eu
+set -eu -o pipefail
 
-readonly NGINX_CONFIG_DIR="/home/yellowtent/setup/configs/nginx" # do not reuse configs since it will be removed by installer
 readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly BOX_SRC_DIR="/home/yellowtent/box"
+readonly DATA_DIR="/home/yellowtent/data"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
 
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
 # copy the website
 rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-rm -rf "${NGINX_CONFIG_DIR}" && mkdir -p "${NGINX_CONFIG_DIR}"
-sed -e "s|##SETUP_WEBSITE_DIR##|${SETUP_WEBSITE_DIR}|" "${script_dir}/splash/nginx/nginx.conf" > "${NGINX_CONFIG_DIR}/nginx.conf"
-cp "${script_dir}/splash/nginx/mime.types" "${NGINX_CONFIG_DIR}/mime.types"
-mkdir -p "${NGINX_CONFIG_DIR}/cert"
-echo "${arg_tls_cert}" > "${NGINX_CONFIG_DIR}/cert/host.cert"
-echo "${arg_tls_key}" > "${NGINX_CONFIG_DIR}/cert/host.key"
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+    rm -f ${DATA_DIR}/nginx/applications/*
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+else
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+fi
 
-# link in the new nginx config
-unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
-ln -s "${NGINX_CONFIG_DIR}" /etc/nginx
-
-touch "${SETUP_WEBSITE_DIR}/progress.json"
+echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
 
 nginx -s reload

setup/start.sh

@@ -1,196 +1,176 @@
 #!/bin/bash
 
-# Count installer files so that we can correlate install and postinstall logs
-install_count=$(find /var/log/cloudron -name "installer*" | wc -l)
-exec > >(tee "/var/log/cloudron/start-$install_count.log")
-exec 2>&1
-
-set -eux
+set -eu -o pipefail
 
 echo "==== Cloudron Start ===="
 
 readonly USER="yellowtent"
-# NOTE: Do NOT use BOX_SRC_DIR for accessing code and config files. This script will be run from a temp directory
-# and the whole code will relocated to BOX_SRC_DIR by the installer. Use paths relative to script_dir or box_src_tmp_dir
 readonly BOX_SRC_DIR="/home/${USER}/box"
 readonly DATA_DIR="/home/${USER}/data"
 readonly CONFIG_DIR="/home/${USER}/configs"
-readonly MAIL_SERVER_IP="172.17.120.120" # hardcoded in haraka container
 readonly SETUP_PROGRESS_JSON="/home/yellowtent/setup/website/progress.json"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-box_src_tmp_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
-admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "admin.${arg_fqdn}" ||  echo "admin-${arg_fqdn}")
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
+readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
 
 set_progress() {
-    local progress="$1"
+    local percent="$1"
     local message="$2"
 
-    echo "==== ${message} ===="
-    (echo "{ \"progress\": \"${progress}\", \"message\": \"${message}\" }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
+    echo "==== ${percent} - ${message} ===="
+    (echo "{ \"update\": { \"percent\": \"${percent}\", \"message\": \"${message}\" }, \"backup\": {} }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
 }
 
-set_progress "5" "Configuring Sudoers file"
-cat > /etc/sudoers.d/yellowtent <<EOF
-Defaults!${BOX_SRC_DIR}/src/scripts/rmappdir.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/rmappdir.sh
+set_progress "1" "Create container"
+$script_dir/container.sh
 
-Defaults!${BOX_SRC_DIR}/src/scripts/reloadnginx.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reloadnginx.sh
+set_progress "10" "Ensuring directories"
+# keep these in sync with paths.js
+[[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
+mkdir -p "${DATA_DIR}/box/appicons"
+mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/graphite"
 
-Defaults!${BOX_SRC_DIR}/src/scripts/backup.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/backup.sh
+mkdir -p "${DATA_DIR}/snapshots"
+mkdir -p "${DATA_DIR}/addons"
+mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
 
-Defaults!${BOX_SRC_DIR}/src/scripts/reboot.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reboot.sh
+# bookkeep the version as part of data
+echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
 
-Defaults!${BOX_SRC_DIR}/src/scripts/reloadcollectd.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reloadcollectd.sh
+# remove old snapshots. if we do want to keep this around, we will have to fix the chown -R below
+# which currently fails because these are readonly fs
+echo "Cleaning up snapshots"
+find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
 
-EOF
+readonly mysql_root_password="password"
+mysqladmin -u root -ppassword password password # reset default root password
+mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
 
-set_progress "10" "Migrating data"
+if [[ -n "${arg_restore_url}" ]]; then
+    set_progress "15" "Downloading restore data"
+
+    echo "Downloading backup: ${arg_restore_url} and key: ${arg_restore_key}"
+
+    while true; do
+        if $curl -L "${arg_restore_url}" | openssl aes-256-cbc -d -pass "pass:${arg_restore_key}" | tar -zxf - -C "${DATA_DIR}/box"; then break; fi
+        echo "Failed to download data, trying again"
+    done
+
+    set_progress "21" "Setting up MySQL"
+    if [[ -f "${DATA_DIR}/box/box.mysqldump" ]]; then
+        echo "Importing existing database into MySQL"
+        mysql -u root -p${mysql_root_password} box < "${DATA_DIR}/box/box.mysqldump"
+    fi
+fi
+
+set_progress "25" "Migrating data"
 sudo -u "${USER}" -H bash <<EOF
-set -eux
-cd "${box_src_tmp_dir}"
-PATH="${PATH}:${box_src_tmp_dir}/node_modules/.bin" npm run-script migrate_data
+set -eu
+cd "${BOX_SRC_DIR}"
+BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
 EOF
 
-set_progress "15" "Setup nginx"
-nginx_config_dir="${CONFIG_DIR}/nginx"
-nginx_appconfig_dir="${CONFIG_DIR}/nginx/applications"
+set_progress "28" "Setup collectd"
+cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
+service collectd restart
 
-# copy nginx config
-mkdir -p "${nginx_appconfig_dir}"
-cp "${script_dir}/start/nginx/nginx.conf" "${nginx_config_dir}/nginx.conf"
-cp "${script_dir}/start/nginx/mime.types" "${nginx_config_dir}/mime.types"
-touch "${nginx_config_dir}/naked_domain.conf"
-sed -e "s/##ADMIN_FQDN##/${admin_fqdn}/" -e "s|##BOX_SRC_DIR##|${BOX_SRC_DIR}|" "${script_dir}/start/nginx/admin.conf_template" > "${nginx_appconfig_dir}/admin.conf"
+set_progress "30" "Setup nginx"
+# setup naked domain to use admin by default. app restoration will overwrite this config
+mkdir -p "${DATA_DIR}/nginx/applications"
+cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"
 
-certificate_dir="${nginx_config_dir}/cert"
-mkdir -p "${certificate_dir}"
-echo "${arg_tls_cert}" > ${certificate_dir}/host.cert
-echo "${arg_tls_key}" > ${certificate_dir}/host.key
+# generate the main nginx config file
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs" \
+    -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
 
-# link nginx config to system config
-unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
-ln -s "${nginx_config_dir}" /etc/nginx
+# generate these for update code paths as well to overwrite splash
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
-chown "${USER}:${USER}" -R "/home/${USER}"
+mkdir -p "${DATA_DIR}/nginx/cert"
+echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
+echo "${arg_tls_key}" > ${DATA_DIR}/nginx/cert/host.key
 
-set_progress "20" "Removing existing container"
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
+set_progress "33" "Changing ownership"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
 
-set_progress "30" "Setup collectd and graphite"
-${script_dir}/start/setup_collectd.sh
+set_progress "40" "Setting up infra"
+${script_dir}/start/setup_infra.sh "${arg_fqdn}"
 
-set_progress "40" "Setup haraka mail relay"
-docker rm -f haraka || true
-docker pull girish/haraka:0.1 || true # this line is for dev convenience since it's already part of base image
-haraka_container_id=$(docker run --restart=always -d --name="haraka" --cap-add="NET_ADMIN"\
-    -p 127.0.0.1:25:25 \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${CONFIG_DIR}/haraka:/app/data" \
-    girish/haraka:0.1)
-echo "Haraka container id: ${haraka_container_id}"
-# Every docker restart results in a new IP. Give our mail server a
-# static IP. Alternately, we need to link the mail container with
-# all our apps
-# This IP is set by the haraka container on every start and the firewall
-# allows connect to port 25. The ping gets the ARP lookup working
-echo "Checking connectivity to haraka(${MAIL_SERVER_IP})"
-if ! ping -c 20 "${MAIL_SERVER_IP}"; then
-    echo "Could not connect to mail server"
-fi
-
-set_progress "50" "Setup MySQL addon"
-docker rm -f mysql || true
-mysql_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-docker pull girish/mysql:0.1 || true # this line for dev convenience since it's already part of base image
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -p 127.0.0.1:3306:3306 \
-    -h "${arg_fqdn}" \
-    -e "MYSQL_ROOT_PASSWORD=${mysql_root_password}" \
-    -e "MYSQL_ROOT_HOST=${docker0_ip}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    girish/mysql:0.1)
-echo "MySQL container id: ${mysql_container_id}"
-
-set_progress "60" "Setup Postgres addon"
-docker rm -f postgresql || true
-postgresql_root_password=$(pwgen -1 -s)
-docker pull girish/postgresql:0.1 || true # this line for dev convenience since it's already part of base image
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -p 127.0.0.1:5432:5432 \
-    -h "${arg_fqdn}" \
-    -e "POSTGRESQL_ROOT_PASSWORD=${postgresql_root_password}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/mysql" \
-    girish/postgresql:0.1)
-echo "PostgreSQL container id: ${postgresql_container_id}"
-
-set_progress "70" "Pulling Redis addon"
-docker pull girish/redis:0.1 || true # this line for dev convenience since it's already part of base image
-
-set_progress "80" "Creating cloudron.conf"
-cloudron_sqlite="${DATA_DIR}/cloudron.sqlite"
-admin_origin="https://${admin_fqdn}"
+set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
-set -eux
+set -eu
 echo "Creating cloudron.conf"
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
     "token": "${arg_token}",
-    "appServerUrl": "${arg_app_server_url}",
+    "apiServerOrigin": "${arg_api_server_origin}",
+    "webServerOrigin": "${arg_web_server_origin}",
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "mailServer": "${MAIL_SERVER_IP}",
-    "mailUsername": "admin@${arg_fqdn}",
-    "addons": {
-        "mysql": {
-            "rootPassword": "${mysql_root_password}"
-        },
-        "postgresql": {
-            "rootPassword": "${postgresql_root_password}"
-        }
+    "adminEmail": "admin@${arg_fqdn}",
+    "database": {
+        "hostname": "localhost",
+        "username": "root",
+        "password": "${mysql_root_password}",
+        "port": 3306,
+        "name": "box"
     }
 }
 CONF_END
 
-echo "Marking apps for restore"
-# TODO: do not auto-start stopped containers (httpPort might need fixing to start them)
-sqlite3 "${cloudron_sqlite}" 'UPDATE apps SET installationState = "pending_restore", healthy = NULL, runState = NULL, containerId = NULL, httpPort = NULL, installationProgress = NULL'
-
-# Add webadmin oauth client
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,profile,users,apps,settings,roleAdmin"
-ADMIN_ID=$(cat /proc/sys/kernel/random/uuid)
-sqlite3 "${cloudron_sqlite}" "INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES (\"\$ADMIN_ID\", \"webadmin\", \"cid-webadmin\", \"secret-webadmin\", \"WebAdmin\", \"${admin_origin}\", \"\$ADMIN_SCOPES\")"
-
+echo "Creating config.json for webadmin"
+cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
+{
+    "webServerOrigin": "${arg_web_server_origin}"
+}
+CONF_END
 EOF
 
-# bookkeep the version as part of data
-echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/version"
+# Add webadmin oauth client
+# The domain might have changed, therefor we have to update the record
+# !!! This needs to be in sync with the webadmin, specifically login_callback.js
+echo "Add webadmin oauth cient"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "90" "Setup supervisord"
-${script_dir}/start/setup_supervisord.sh
+echo "Add localhost test oauth cient"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "95" "Reloading supervisor"
-${script_dir}/start/reload_supervisord.sh
+set_progress "80" "Reloading supervisor"
+# looks like restarting supervisor completely is the only way to reload it
+service supervisor stop || true
 
-set_progress "99" "Reloading nginx"
+echo -n "Waiting for supervisord to stop"
+while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
+    echo -n "."
+    sleep 1
+done
+echo ""
+
+echo "Starting supervisor"
+
+service supervisor start
+
+sleep 2 # give supervisor sometime to start the processes
+
+set_progress "85" "Reloading nginx"
 nginx -s reload
 
 set_progress "100" "Done"

setup/start/collectd.conf

@@ -52,20 +52,20 @@ Interval 20
 # accessed.                                                                  #
 ##############################################################################
 
-#LoadPlugin logfile
-LoadPlugin syslog
+LoadPlugin logfile
+#LoadPlugin syslog
 
-#<Plugin logfile>
-#   LogLevel "info"
-#   File STDOUT
-#   Timestamp true
-#   PrintSeverity false
-#</Plugin>
-
-<Plugin syslog>
-    LogLevel info
+<Plugin logfile>
+   LogLevel "info"
+   File "/var/log/collectd.log"
+   Timestamp true
+   PrintSeverity false
 </Plugin>
 
+#<Plugin syslog>
+#    LogLevel info
+#</Plugin>
+
 ##############################################################################
 # LoadPlugin section                                                         #
 #----------------------------------------------------------------------------#
@@ -96,7 +96,7 @@ LoadPlugin df
 #LoadPlugin entropy
 #LoadPlugin ethstat
 #LoadPlugin exec
-LoadPlugin filecount
+#LoadPlugin filecount
 #LoadPlugin fscache
 #LoadPlugin gmond
 #LoadPlugin hddtemp

setup/start/nginx/admin.conf_template

@@ -1,33 +0,0 @@
-server {
-    listen       443;
-    server_name  ##ADMIN_FQDN##;
-
-    ssl                  on;
-    # paths are relative to prefix and not to this file
-    ssl_certificate      cert/host.cert;
-    ssl_certificate_key  cert/host.key;
-
-
-    ssl_session_timeout  5m;
-
-    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
-    ssl_prefer_server_ciphers on;
-
-    location /api/ {
-        proxy_pass   http://127.0.0.1:3000;
-        client_max_body_size 1m;
-    }
-
-    # graphite paths
-    location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
-        proxy_pass   http://127.0.0.1:8000;
-        client_max_body_size 1m;
-    }
-
-    location / {
-        root   ##BOX_SRC_DIR##/webadmin/dist/;
-        index  index.html index.htm;
-    }
-}
-

setup/start/nginx/appconfig.ejs

@@ -0,0 +1,113 @@
+# http://nginx.org/en/docs/http/websocket.html
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+server {
+    listen       443;
+    server_name  <%= vhost %>;
+
+    ssl                  on;
+    # paths are relative to prefix and not to this file
+    ssl_certificate      cert/host.cert;
+    ssl_certificate_key  cert/host.key;
+    ssl_session_timeout  5m;
+    ssl_session_cache shared:SSL:50m;
+
+    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
+    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
+    # https://cipherli.st/
+    ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
+
+    proxy_http_version 1.1;
+    proxy_intercept_errors on;
+    proxy_read_timeout       3500;
+    proxy_connect_timeout    3250;
+
+    proxy_set_header   Host               $host;
+    proxy_set_header   X-Forwarded-For    $remote_addr;
+    proxy_set_header   X-Forwarded-Host   $host;
+    proxy_set_header   X-Forwarded-Proto  https;
+
+    # upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    error_page 500 502 503 504 @appstatus;
+    location @appstatus {
+        return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
+    }
+
+    location / {
+        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+        proxy_buffer_size       128k;
+        proxy_buffers           4 256k;
+        proxy_busy_buffers_size 256k;
+
+        # Disable check to allow unlimited body sizes
+        client_max_body_size 0;
+
+<% if ( endpoint === 'admin' ) { %>
+        location /api/ {
+            proxy_pass   http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
+
+        # graphite paths
+        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+            proxy_pass   http://127.0.0.1:8000;
+            client_max_body_size 1m;
+        }
+
+        location / {
+            root   <%= sourceDir %>/webadmin/dist;
+            index  index.html index.htm;
+        }
+
+<% } else if ( endpoint === 'oauthproxy' ) { %>
+        proxy_pass http://127.0.0.1:4000;
+        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
+<% } else if ( endpoint === 'app' ) { %>
+        proxy_pass http://127.0.0.1:<%= port %>;
+<% } else if ( endpoint === 'splash' ) { %>
+        root   <%= sourceDir %>;
+
+        error_page 503 /update.html;
+
+        location /update.html {
+            add_header Cache-Control no-cache;
+        }
+
+        location /theme.css {
+            add_header Cache-Control no-cache;
+        }
+
+        location /3rdparty/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /js/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /progress.json {
+            add_header Cache-Control no-cache;
+        }
+
+        location /api/v1/cloudron/progress {
+            add_header Cache-Control no-cache;
+            default_type application/json;
+            alias <%= sourceDir %>/progress.json;
+        }
+
+        location / {
+            return 503;
+        }
+<% } %>
+    }
+}
+

setup/start/nginx/nginx.ejs

@@ -17,6 +17,9 @@ http {
         '"$request" $status $body_bytes_sent $request_time '
         '"$http_referer" "$http_user_agent"';
 
+    # required for long host names
+    server_names_hash_bucket_size 128;
+
     access_log access.log combined2;
 
     sendfile        on;
@@ -49,11 +52,16 @@ http {
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
+        error_page 404 = @fallback;
+        location @fallback {
+            internal;
+            root <%= sourceDir %>/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
         return 404;
     }
 
-    include naked_domain.conf;
-
     include applications/*.conf;
 }
 

setup/start/reload_supervisord.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# looks like restarting supervisor completely is the only way to reload it
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
-

setup/start/setup_collectd.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly GRAPHITE_DIR="/home/yellowtent/data/graphite"
-readonly COLLECTD_CONFIG_DIR="/home/yellowtent/configs/collectd"
-readonly COLLECTD_APPCONFIG_DIR="${COLLECTD_CONFIG_DIR}/collectd.conf.d"
-
-readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-mkdir -p "${GRAPHITE_DIR}"
-
-docker rm -f graphite || true
-docker pull girish/graphite:0.2
-docker run --restart=always -d --name="graphite" \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${GRAPHITE_DIR}:/app/data" girish/graphite:0.2
-
-mkdir -p "${COLLECTD_APPCONFIG_DIR}"
-cp -r "${script_dir}/collectd/collectd.conf" "${COLLECTD_CONFIG_DIR}/collectd.conf"
-rm -rf /etc/collectd
-ln -sfF "${COLLECTD_CONFIG_DIR}" /etc/collectd
-chown -R yellowtent.yellowtent "${COLLECTD_CONFIG_DIR}"
-
-update-rc.d -f collectd defaults
-/etc/init.d/collectd restart
-

setup/start/setup_infra.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly DATA_DIR="/home/yellowtent/data"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
+
+arg_fqdn="$1"
+
+# removing containers ensures containers are launched with latest config updates
+# restore code in appatask does not delete old containers
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
+    echo "Infrastructure is upto date"
+    exit 0
+fi
+
+echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
+
+existing_containers=$(docker ps -qa)
+echo "Remove containers: ${existing_containers}"
+if [[ -n "${existing_containers}" ]]; then
+    echo "${existing_containers}" | xargs docker rm -f
+fi
+
+# graphite
+docker run --restart=always -d --name="graphite" \
+    -p 127.0.0.1:2003:2003 \
+    -p 127.0.0.1:2004:2004 \
+    -p 127.0.0.1:8000:8000 \
+    -v "${DATA_DIR}/graphite:/app/data" cloudron/graphite:0.3.1
+
+# mail
+mail_container_id=$(docker run --restart=always -d --name="mail" \
+    -p 127.0.0.1:25:25 \
+    -h "${arg_fqdn}" \
+    -e "DOMAIN_NAME=${arg_fqdn}" \
+    -v "${DATA_DIR}/box/mail:/app/data" \
+    cloudron/mail:0.3.0)
+echo "Mail container id: ${mail_container_id}"
+
+# mysql
+mysql_addon_root_password=$(pwgen -1 -s)
+docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
+cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
+readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
+readonly MYSQL_ROOT_HOST='${docker0_ip}'
+EOF
+mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
+    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:r" \
+    cloudron/mysql:0.3.0)
+echo "MySQL container id: ${mysql_container_id}"
+
+# postgresql
+postgresql_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
+readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
+EOF
+postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
+    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:r" \
+    cloudron/postgresql:0.3.0)
+echo "PostgreSQL container id: ${postgresql_container_id}"
+
+# mongodb
+mongodb_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
+readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
+EOF
+mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
+    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:r" \
+    cloudron/mongodb:0.3.0)
+echo "Mongodb container id: ${mongodb_container_id}"
+
+if [[ "${infra_version}" == "none" ]]; then
+    # if no existing infra was found (for new and restoring cloudons), download app backups
+    echo "Marking installed apps for restore"
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore" WHERE installationState = "installed"' box
+else
+    # if existing infra was found, just mark apps for reconfiguration
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure" WHERE installationState = "installed"' box
+fi
+
+echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"
+

setup/start/setup_supervisord.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly BOX_SRC_DIR="/home/yellowtent/box"
-readonly DATA_DIR="/home/yellowtent/data"
-
-readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-rm -rf /etc/supervisor
-mkdir -p /etc/supervisor/conf.d
-cp "${script_dir}/supervisord/supervisord.conf" /etc/supervisor/
-
-echo "Writing supervisor configs..."
-
-cat > /etc/supervisor/conf.d/box.conf <<EOF
-[program:box]
-command=/usr/bin/node "${BOX_SRC_DIR}/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",NODE_ENV="cloudron"
-EOF
-
-cat > /etc/supervisor/conf.d/oauthproxy.conf <<EOF
-[program:oauthproxy]
-command=/usr/bin/node "${BOX_SRC_DIR}/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/proxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
-EOF
-
-cat > /etc/supervisor/conf.d/apphealthtask.conf <<EOF
-[program:apphealthtask]
-command=/usr/bin/node "${BOX_SRC_DIR}/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
-EOF
-

setup/stop.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -eu
+set -eu -o pipefail
 
 echo "Stopping box code"
 

src/addons.js

@@ -1,177 +1,345 @@
 'use strict';
 
-var appdb = require('./appdb.js'),
-    assert = require('assert'),
-    async = require('async'),
-    child_process = require('child_process'),
-    clientdb = require('./clientdb.js'),
-    config = require('../config.js'),
-    DatabaseError = require('./databaseerror.js'),
-    debug = require('debug')('box:addons'),
-    docker = require('./docker.js'),
-    generatePassword = require('password-generator'),
-    MemoryStream = require('memorystream'),
-    os = require('os'),
-    safe = require('safetydance'),
-    util = require('util'),
-    uuid = require('node-uuid');
-
 exports = module.exports = {
     setupAddons: setupAddons,
     teardownAddons: teardownAddons,
+    backupAddons: backupAddons,
+    restoreAddons: restoreAddons,
+
     getEnvironment: getEnvironment,
+    getLinksSync: getLinksSync,
+    getBindsSync: getBindsSync,
 
     // exported for testing
     _allocateOAuthCredentials: allocateOAuthCredentials,
     _removeOAuthCredentials: removeOAuthCredentials
 };
 
+var appdb = require('./appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    child_process = require('child_process'),
+    clientdb = require('./clientdb.js'),
+    config = require('./config.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:addons'),
+    docker = require('./docker.js'),
+    fs = require('fs'),
+    generatePassword = require('password-generator'),
+    hat = require('hat'),
+    MemoryStream = require('memorystream'),
+    once = require('once'),
+    os = require('os'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    spawn = child_process.spawn,
+    tokendb = require('./tokendb.js'),
+    util = require('util'),
+    uuid = require('node-uuid'),
+    vbox = require('./vbox.js'),
+    _ = require('underscore');
+
+var NOOP = function (app, callback) { return callback(); };
+
+// setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
+// teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
     oauth: {
         setup: allocateOAuthCredentials,
-        teardown: removeOAuthCredentials
+        teardown: removeOAuthCredentials,
+        backup: NOOP,
+        restore: allocateOAuthCredentials
+    },
+    token: {
+        setup: allocateAccessToken,
+        teardown: removeAccessToken,
+        backup: NOOP,
+        restore: allocateAccessToken
+    },
+    ldap: {
+        setup: setupLdap,
+        teardown: teardownLdap,
+        backup: NOOP,
+        restore: setupLdap
     },
     sendmail: {
         setup: setupSendMail,
-        teardown: teardownSendMail
+        teardown: teardownSendMail,
+        backup: NOOP,
+        restore: setupSendMail
     },
     mysql: {
         setup: setupMySql,
-        teardown: teardownMySql
+        teardown: teardownMySql,
+        backup: backupMySql,
+        restore: restoreMySql,
     },
     postgresql: {
         setup: setupPostgreSql,
-        teardown: teardownPostgreSql
+        teardown: teardownPostgreSql,
+        backup: backupPostgreSql,
+        restore: restorePostgreSql
+    },
+    mongodb: {
+        setup: setupMongoDb,
+        teardown: teardownMongoDb,
+        backup: backupMongoDb,
+        restore: restoreMongoDb
     },
     redis: {
         setup: setupRedis,
-        teardown: teardownRedis
+        teardown: teardownRedis,
+        backup: NOOP, // no backup because we store redis as part of app's volume
+        restore: setupRedis // same thing
+    },
+    localstorage: {
+        setup: NOOP, // docker creates the directory for us
+        teardown: NOOP,
+        backup: NOOP, // no backup because it's already inside app data
+        restore: NOOP
+    },
+    _docker: {
+        setup: NOOP,
+        teardown: NOOP,
+        backup: NOOP,
+        restore: NOOP
     }
 };
 
-function forwardFromHostToVirtualBox(rulename, port) {
-    if (os.platform() === 'darwin') {
-        debug('Setting up VirtualBox port forwarding for '+ rulename + ' at ' + port);
-        child_process.exec(
-            'VBoxManage controlvm boot2docker-vm natpf1 delete ' + rulename + ';' +
-            'VBoxManage controlvm boot2docker-vm natpf1 ' + rulename + ',tcp,127.0.0.1,' + port + ',,' + port);
-    }
+var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh');
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-function unforwardFromHostToVirtualBox(rulename) {
-    if (os.platform() === 'darwin') {
-        debug('Removing VirtualBox port forwarding for '+ rulename);
-        child_process.exec('VBoxManage controlvm boot2docker-vm natpf1 delete ' + rulename);
-    }
-}
+function setupAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-function setupAddons(app, callback) {
-    assert(typeof app === 'object');
-    assert(!app.manifest.addons || util.isArray(app.manifest.addons));
-    assert(typeof callback === 'function');
+    if (!addons) return callback(null);
 
-    if (!app.manifest.addons) return callback(null);
+    debugApp(app, 'setupAddons: Settings up %j', Object.keys(addons));
 
-    async.eachSeries(app.manifest.addons, function iterator(addon, iteratorCallback) {
+    async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
+        debugApp(app, 'Setting up addon %s', addon);
+
         KNOWN_ADDONS[addon].setup(app, iteratorCallback);
     }, callback);
 }
 
-function teardownAddons(app, callback) {
-    assert(typeof app === 'object');
-    assert(!app.manifest.addons || util.isArray(app.manifest.addons));
-    assert(typeof callback === 'function');
+function teardownAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    if (!app.manifest.addons) return callback(null);
+    if (!addons) return callback(null);
 
-    async.eachSeries(app.manifest.addons, function iterator(addon, iteratorCallback) {
+    debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
+        debugApp(app, 'Tearing down addon %s', addon);
+
         KNOWN_ADDONS[addon].teardown(app, iteratorCallback);
     }, callback);
 }
 
-function getEnvironment(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+function backupAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    appdb.getAddonConfigByAppId(appId, callback);
+    debugApp(app, 'backupAddons');
+
+    if (!addons) return callback(null);
+
+    debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+
+        KNOWN_ADDONS[addon].backup(app, iteratorCallback);
+    }, callback);
+}
+
+function restoreAddons(app, addons, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'restoreAddons');
+
+    if (!addons) return callback(null);
+
+    debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons));
+
+    async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
+        if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
+
+        KNOWN_ADDONS[addon].restore(app, iteratorCallback);
+    }, callback);
+}
+
+function getEnvironment(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAddonConfigByAppId(app.id, callback);
+}
+
+function getLinksSync(app, addons) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+
+    var links = [ ];
+
+    if (!addons) return links;
+
+    for (var addon in addons) {
+        switch (addon) {
+        case 'mysql': links.push('mysql:mysql'); break;
+        case 'postgresql': links.push('postgresql:postgresql'); break;
+        case 'sendmail': links.push('mail:mail'); break;
+        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
+        case 'mongodb': links.push('mongodb:mongodb'); break;
+        default: break;
+        }
+    }
+
+    return links;
+}
+
+function getBindsSync(app, addons) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+
+    var binds = [ ];
+
+    if (!addons) return binds;
+
+    for (var addon in addons) {
+        switch (addon) {
+        case '_docker': binds.push('/var/run/docker.sock:/var/run/docker.sock:rw'); break;
+        case 'localstorage': binds.push(path.join(paths.DATA_DIR, app.id, 'data') + ':/app/data:rw'); break;
+        default: break;
+        }
+    }
+
+    return binds;
 }
 
 function allocateOAuthCredentials(app, callback) {
-    assert(typeof app === 'object');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    var id = uuid.v4();
     var appId = app.id;
-    var clientId = 'cid-' + uuid.v4();
-    var clientSecret = uuid.v4();
-    var name = app.manifest.title;
+    var id = 'cid-addon-' + uuid.v4();
+    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile,roleUser';
 
-    debug('allocateOAuthCredentials: id:%s clientId:%s clientSecret:%s name:%s', id, clientId, clientSecret, name);
+    debugApp(app, 'allocateOAuthCredentials: id:%s clientSecret:%s', id, clientSecret);
 
-    clientdb.add(id, appId, clientId, clientSecret, name, redirectURI, scope, function (error) {
-        if (error) return callback(error);
+    clientdb.delByAppId('addon-' + appId, function (error) { // remove existing creds
+        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
 
-        var env = [
-            'OAUTH_CLIENT_ID=' + clientId,
-            'OAUTH_CLIENT_SECRET=' + clientSecret
-        ];
+        clientdb.add(id, 'addon-' + appId, clientSecret, redirectURI, scope, function (error) {
+            if (error) return callback(error);
 
-        appdb.setAddonConfig(appId, 'oauth', env, callback);
+            var env = [
+                'OAUTH_CLIENT_ID=' + id,
+                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_ORIGIN=' + config.adminOrigin()
+            ];
+
+            debugApp(app, 'Setting oauth addon config to %j', env);
+
+            appdb.setAddonConfig(appId, 'oauth', env, callback);
+        });
     });
 }
 
 function removeOAuthCredentials(app, callback) {
-    assert(typeof app === 'object');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    debug('removeOAuthCredentials: %s', app.id);
+    debugApp(app, 'removeOAuthCredentials');
 
-    clientdb.delByAppId(app.id, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null);
-        if (error) console.error(error);
+    clientdb.delByAppId('addon-' + app.id, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
 }
 
-function setupSendMail(app, callback) {
-    assert(typeof app === 'object');
-    assert(typeof callback === 'function');
+function setupLdap(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'MAIL_SERVER=' + config.get('mailServer'),
-        'MAIL_USERNAME=' + app.location,
+        'LDAP_SERVER=172.17.42.1',
+        'LDAP_PORT=3002',
+        'LDAP_URL=ldap://172.17.42.1:3002',
+        'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
+        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron'
+    ];
+
+    debugApp(app, 'Setting up LDAP');
+
+    appdb.setAddonConfig(app.id, 'ldap', env, callback);
+}
+
+function teardownLdap(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down LDAP');
+
+    appdb.unsetAddonConfig(app.id, 'ldap', callback);
+}
+
+function setupSendMail(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=25',
+        'MAIL_SMTP_USERNAME=' + (app.location || app.id), // use app.id for bare domains
         'MAIL_DOMAIN=' + config.fqdn()
     ];
 
-    debug('Setting up sendmail for %s', app.id);
+    debugApp(app, 'Setting up sendmail');
 
     appdb.setAddonConfig(app.id, 'sendmail', env, callback);
 }
 
 function teardownSendMail(app, callback) {
-    assert(typeof app === 'object');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    debug('Tearing down sendmail for %s', app.id);
+    debugApp(app, 'Tearing down sendmail');
 
     appdb.unsetAddonConfig(app.id, 'sendmail', callback);
 }
 
 function setupMySql(app, callback) {
-    assert(typeof app === 'object');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    debug('Setting up mysql for %s', app.id);
+    debugApp(app, 'Setting up mysql');
 
     var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'add', config.get('addons.mysql.rootPassword'), app.id ];
+    var cmd = [ '/addons/mysql/service.sh', 'add', app.id ];
 
     container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
         if (error) return callback(error);
@@ -183,7 +351,7 @@ function setupMySql(app, callback) {
             var stderr = new MemoryStream();
 
             execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debug(data); }); // set -e output
+            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
 
             var chunks = [ ];
             stdout.on('data', function (chunk) { chunks.push(chunk); });
@@ -191,7 +359,7 @@ function setupMySql(app, callback) {
             stream.on('error', callback);
             stream.on('end', function () {
                 var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debug('Setting mysql addon config to %j', env);
+                debugApp(app, 'Setting mysql addon config to %j', env);
                 appdb.setAddonConfig(app.id, 'mysql', env, callback);
             });
         });
@@ -200,14 +368,14 @@ function setupMySql(app, callback) {
 
 function teardownMySql(app, callback) {
     var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'remove', config.get('addons.mysql.rootPassword'), app.id ];
+    var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
 
-    debug('Tearing down mysql for %s', app.id);
+    debugApp(app, 'Tearing down mysql');
 
-    container.exec({ Cmd: cmd }, function (error, execContainer) {
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
         if (error) return callback(error);
 
-        execContainer.start({ stream: true, stdout: true, stderr: true }, function (error, stream) {
+        execContainer.start(function (error, stream) {
             if (error) return callback(error);
 
             var data = '';
@@ -220,14 +388,58 @@ function teardownMySql(app, callback) {
     });
 }
 
-function setupPostgreSql(app, callback) {
-    assert(typeof app === 'object');
-    assert(typeof callback === 'function');
+function backupMySql(app, callback) {
+    debugApp(app, 'Backing up mysql');
 
-    debug('Setting up postgresql for %s', app.id);
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
+    output.on('error', callback);
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', 'backup', app.id ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
+        if (!callback.called) callback(code ? 'backupMySql failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(output);
+    cp.stderr.pipe(process.stderr);
+}
+
+function restoreMySql(app, callback) {
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    setupMySql(app, function (error) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreMySql');
+
+        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
+        input.on('error', callback);
+
+        // cannot get this to work through docker.exec
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', 'restore', app.id ]);
+        cp.on('error', callback);
+        cp.on('exit', function (code, signal) {
+            debugApp(app, 'restoreMySql: done %s %s', code, signal);
+            if (!callback.called) callback(code ? 'restoreMySql failed with status ' + code : null);
+        });
+
+        cp.stdout.pipe(process.stdout);
+        cp.stderr.pipe(process.stderr);
+        input.pipe(cp.stdin).on('error', callback);
+    });
+}
+
+function setupPostgreSql(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up postgresql');
 
     var container = docker.getContainer('postgresql');
-    var cmd = [ '/addons/postgresql/service.sh', 'add', config.get('addons.postgresql.rootPassword'), app.id ];
+    var cmd = [ '/addons/postgresql/service.sh', 'add', app.id ];
 
     container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
         if (error) return callback(error);
@@ -239,7 +451,7 @@ function setupPostgreSql(app, callback) {
             var stderr = new MemoryStream();
 
             execContainer.modem.demuxStream(stream, stdout, stderr);
-            stderr.on('data', function (data) { debug(data); }); // set -e output
+            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
 
             var chunks = [ ];
             stdout.on('data', function (chunk) { chunks.push(chunk); });
@@ -247,7 +459,7 @@ function setupPostgreSql(app, callback) {
             stream.on('error', callback);
             stream.on('end', function () {
                 var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
-                debug('Setting postgresql addon config to %j', env);
+                debugApp(app, 'Setting postgresql addon config to %j', env);
                 appdb.setAddonConfig(app.id, 'postgresql', env, callback);
             });
         });
@@ -256,14 +468,14 @@ function setupPostgreSql(app, callback) {
 
 function teardownPostgreSql(app, callback) {
     var container = docker.getContainer('postgresql');
-    var cmd = [ '/addons/postgresql/service.sh', 'remove', config.get('addons.postgresql.rootPassword'), app.id ];
+    var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
 
-    debug('Tearing down postgresql for %s', app.id);
+    debugApp(app, 'Tearing down postgresql');
 
-    container.exec({ Cmd: cmd }, function (error, execContainer) {
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
         if (error) return callback(error);
 
-        execContainer.start({ stream: true, stdout: true, stderr: true }, function (error, stream) {
+        execContainer.start(function (error, stream) {
             if (error) return callback(error);
 
             var data = '';
@@ -276,56 +488,227 @@ function teardownPostgreSql(app, callback) {
     });
 }
 
+function backupPostgreSql(app, callback) {
+    debugApp(app, 'Backing up postgresql');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
+    output.on('error', callback);
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'postgresql', '/addons/postgresql/service.sh', 'backup', app.id ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupPostgreSql: done %s %s', code, signal);
+        if (!callback.called) callback(code ? 'backupPostgreSql failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(output);
+    cp.stderr.pipe(process.stderr);
+}
+
+function restorePostgreSql(app, callback) {
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    setupPostgreSql(app, function (error) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restorePostgreSql');
+
+        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'postgresqldump'));
+        input.on('error', callback);
+
+        // cannot get this to work through docker.exec
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'postgresql', '/addons/postgresql/service.sh', 'restore', app.id ]);
+        cp.on('error', callback);
+        cp.on('exit', function (code, signal) {
+            debugApp(app, 'restorePostgreSql: done %s %s', code, signal);
+            if (!callback.called) callback(code ? 'restorePostgreSql failed with status ' + code : null);
+        });
+
+        cp.stdout.pipe(process.stdout);
+        cp.stderr.pipe(process.stderr);
+        input.pipe(cp.stdin).on('error', callback);
+    });
+}
+
+function setupMongoDb(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up mongodb');
+
+    var container = docker.getContainer('mongodb');
+    var cmd = [ '/addons/mongodb/service.sh', 'add', app.id ];
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var stdout = new MemoryStream();
+            var stderr = new MemoryStream();
+
+            execContainer.modem.demuxStream(stream, stdout, stderr);
+            stderr.on('data', function (data) { debugApp(app, data.toString('utf8')); }); // set -e output
+
+            var chunks = [ ];
+            stdout.on('data', function (chunk) { chunks.push(chunk); });
+
+            stream.on('error', callback);
+            stream.on('end', function () {
+                var env = Buffer.concat(chunks).toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+                debugApp(app, 'Setting mongodb addon config to %j', env);
+                appdb.setAddonConfig(app.id, 'mongodb', env, callback);
+            });
+        });
+    });
+}
+
+function teardownMongoDb(app, callback) {
+    var container = docker.getContainer('mongodb');
+    var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
+
+    debugApp(app, 'Tearing down mongodb');
+
+    container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
+        if (error) return callback(error);
+
+        execContainer.start(function (error, stream) {
+            if (error) return callback(error);
+
+            var data = '';
+            stream.on('error', callback);
+            stream.on('data', function (d) { data += d.toString('utf8'); });
+            stream.on('end', function () {
+                appdb.unsetAddonConfig(app.id, 'mongodb', callback);
+            });
+        });
+    });
+}
+
+function backupMongoDb(app, callback) {
+    debugApp(app, 'Backing up mongodb');
+
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
+    output.on('error', callback);
+
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'mongodb', '/addons/mongodb/service.sh', 'backup', app.id ]);
+    cp.on('error', callback);
+    cp.on('exit', function (code, signal) {
+        debugApp(app, 'backupMongoDb: done %s %s', code, signal);
+        if (!callback.called) callback(code ? 'backupMongoDb failed with status ' + code : null);
+    });
+
+    cp.stdout.pipe(output);
+    cp.stderr.pipe(process.stderr);
+}
+
+function restoreMongoDb(app, callback) {
+    callback = once(callback); // ChildProcess exit may or may not be called after error
+
+    setupMongoDb(app, function (error) {
+        if (error) return callback(error);
+
+        debugApp(app, 'restoreMongoDb');
+
+        var input = fs.createReadStream(path.join(paths.DATA_DIR, app.id, 'mongodbdump'));
+        input.on('error', callback);
+
+        // cannot get this to work through docker.exec
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mongodb', '/addons/mongodb/service.sh', 'restore', app.id ]);
+        cp.on('error', callback);
+        cp.on('exit', function (code, signal) {
+            debugApp(app, 'restoreMongoDb: done %s %s', code, signal);
+            if (!callback.called) callback(code ? 'restoreMongoDb failed with status ' + code : null);
+        });
+
+        cp.stdout.pipe(process.stdout);
+        cp.stderr.pipe(process.stderr);
+        input.pipe(cp.stdin).on('error', callback);
+    });
+}
+
+
+function forwardRedisPort(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.getContainer('redis-' + appId).inspect(function (error, data) {
+        if (error) return callback(new Error('Unable to inspect container:' + error));
+
+        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
+        if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
+
+        vbox.forwardFromHostToVirtualBox('redis-' + appId, redisPort);
+
+        return callback(null);
+    });
+}
+
+// Ensures that app's addon redis container is running. Can be called when named container already exists/running
 function setupRedis(app, callback) {
     var redisPassword = generatePassword(64, false /* memorable */);
+    var redisVarsFile = path.join(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
+    var redisDataDir = path.join(paths.DATA_DIR, app.id + '/redis');
+
+    if (!safe.fs.writeFileSync(redisVarsFile, 'REDIS_PASSWORD=' + redisPassword)) {
+        return callback(new Error('Error writing redis config'));
+    }
+
+    if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error));
 
     var createOptions = {
         name: 'redis-' + app.id,
         Hostname: config.appFqdn(app.location),
         Tty: true,
-        Image: 'girish/redis:0.1',
+        Image: 'cloudron/redis:0.3.0',
         Cmd: null,
-        Volumes: { },
-        VolumesFrom: '',
-        Env: [ 'REDIS_PASSWORD=' + redisPassword ]
+        Volumes: {},
+        VolumesFrom: []
     };
 
     var isMac = os.platform() === 'darwin';
 
     var startOptions = {
-        Binds: [ ],
+        Binds: [
+            redisVarsFile + ':/etc/redis/redis_vars.sh:r',
+            redisDataDir + ':/var/lib/redis:rw'
+        ],
         // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work
         // On linux, export to localhost only for testing purposes and not for the app itself
         PortBindings: {
             '6379/tcp': [{ HostPort: '0', HostIp: isMac ? '0.0.0.0' : '127.0.0.1' }]
+        },
+        RestartPolicy: {
+            'Name': 'always',
+            'MaximumRetryCount': 0
         }
     };
 
-    // docker.run does not return until the container ends :/
-    docker.createContainer(createOptions, function (error, container) {
-        if (error) return callback(new Error('Error creating container:' + error));
+    var env = [
+        'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id,
+        'REDIS_PASSWORD=' + redisPassword,
+        'REDIS_HOST=redis-' + app.id,
+        'REDIS_PORT=6379'
+    ];
 
-        debug('Created redis container for %s with id %s', app.id, container.id);
+    var redisContainer = docker.getContainer(createOptions.name);
+    redisContainer.remove({ force: true, v: false }, function (ignoredError) {
+        docker.createContainer(createOptions, function (error) {
+            if (error && error.statusCode !== 409) return callback(error); // if not already created
 
-        container.start(startOptions, function (error) {
-            if (error) return callback(new Error('Error starting container:' + error));
+            redisContainer.start(startOptions, function (error) {
+                if (error && error.statusCode !== 304) return callback(error); // if not already running
 
-            debug('Started redis container for %s with id %s', app.id, container.id);
+                appdb.setAddonConfig(app.id, 'redis', env, function (error) {
+                    if (error) return callback(error);
 
-            container.inspect(function (error, data) {
-                if (error) return callback(new Error('Unable to inspect container:' + error));
-
-                var redisIp = safe.query(data, 'NetworkSettings.IPAddress');
-                if (!redisIp) return callback(new Error('Unable to get container ip'));
- 
-                var redisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
-                if (!redisPort) return callback(new Error('Unable to get container port mapping'));
-
-                forwardFromHostToVirtualBox('redis-' + app.id, redisPort);
-
-                var env = [ 'REDIS_URL=redis://redisuser:' + redisPassword + '@' + redisIp + ':6379' ];
-
-                appdb.setAddonConfig(app.id, 'redis', env, callback);
+                    forwardRedisPort(app.id, callback);
+                });
             });
         });
     });
@@ -340,12 +723,54 @@ function teardownRedis(app, callback) {
    };
 
    container.remove(removeOptions, function (error) {
-       if (error && error.statusCode === 404) return callback(null);
-       if (error) return callback(new Error('Error removing container:' + error));
+       if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
 
-       unforwardFromHostToVirtualBox('redis-' + app.id);
+       vbox.unforwardFromHostToVirtualBox('redis-' + app.id);
 
-       callback(null);
+       safe.fs.unlinkSync(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
+
+        shell.sudo('teardownRedis', [ RMAPPDIR_CMD, app.id + '/redis' ], function (error, stdout, stderr) {
+            if (error) return callback(new Error('Error removing redis data:' + error));
+
+            appdb.unsetAddonConfig(app.id, 'redis', callback);
+        });
    });
 }
 
+function allocateAccessToken(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var token = tokendb.generateToken();
+    var expiresAt = Number.MAX_SAFE_INTEGER;    // basically never expire
+    var scopes = 'profile,users';               // TODO This should be put into the manifest and the user should know those
+    var clientId = '';                          // meaningless for apps so far
+
+   tokendb.delByIdentifier(tokendb.PREFIX_APP + app.id, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
+
+        tokendb.add(token, tokendb.PREFIX_APP + app.id, clientId, expiresAt, scopes, function (error) {
+            if (error) return callback(error);
+
+            var env = [
+                'CLOUDRON_TOKEN=' + token
+            ];
+
+            debugApp(app, 'Setting token addon config to %j', env);
+
+            appdb.setAddonConfig(appId, 'token', env, callback);
+        });
+    });
+}
+
+function removeAccessToken(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delByIdentifier(tokendb.PREFIX_APP + app.id, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+
+        appdb.unsetAddonConfig(app.id, 'token', callback);
+    });
+}
+

src/appdb.js

@@ -2,14 +2,6 @@
 
 'use strict';
 
-var assert = require('assert'),
-    async = require('async'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror'),
-    debug = require('debug')('box:appdb'),
-    safe = require('safetydance'),
-    util = require('util');
-
 exports = module.exports = {
     get: get,
     getBySubdomain: getBySubdomain,
@@ -20,7 +12,6 @@ exports = module.exports = {
     update: update,
     getAll: getAll,
     getPortBindings: getPortBindings,
-    clear: clear,
 
     setAddonConfig: setAddonConfig,
     getAddonConfig: getAddonConfig,
@@ -31,179 +22,199 @@ exports = module.exports = {
     setHealth: setHealth,
     setInstallationCommand: setInstallationCommand,
     setRunCommand: setRunCommand,
-    getAppVersions: getAppVersions,
+    getAppStoreIds: getAppStoreIds,
 
-    // status codes
-    ISTATE_PENDING_INSTALL: 'pending_install',
-    ISTATE_PENDING_CONFIGURE: 'pending_configure',
-    ISTATE_PENDING_UNINSTALL: 'pending_uninstall',
-    ISTATE_PENDING_RESTORE: 'pending_restore',
-    ISTATE_PENDING_UPDATE: 'pending_update',
-    ISTATE_ERROR: 'error',
-    ISTATE_INSTALLED: 'installed',
+    // installation codes (keep in sync in UI)
+    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
+    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
+    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
+    ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
+    ISTATE_PENDING_FORCE_UPDATE: 'pending_force_update', // update from any state preserving data
+    ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app
+    ISTATE_ERROR: 'error', // error executing last pending_* command
+    ISTATE_INSTALLED: 'installed', // app is installed
 
+    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
-    RSTATE_STOPPED: 'stopped', // app stopped by user
-    RSTATE_DEAD: 'dead', // app stopped on it's own
-    RSTATE_ERROR: 'error'
+    RSTATE_STOPPED: 'stopped', // app stopped by use
+
+    HEALTH_HEALTHY: 'healthy',
+    HEALTH_UNHEALTHY: 'unhealthy',
+    HEALTH_ERROR: 'error',
+    HEALTH_DEAD: 'dead',
+
+    _clear: clear
 };
 
-var APPS_FIELDS = [ 'id', 'appStoreId', 'version', 'installationState', 'installationProgress', 'runState',
-    'healthy', 'containerId', 'manifestJson', 'httpPort', 'location', 'dnsRecordId', 'accessRestriction' ].join(',');
+var assert = require('assert'),
+    async = require('async'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    safe = require('safetydance'),
+    util = require('util');
 
-var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.version', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
-    'apps.healthy', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId', 'apps.accessRestriction' ].join(',');
+var APPS_FIELDS = [ 'id', 'appStoreId', 'installationState', 'installationProgress', 'runState',
+    'health', 'containerId', 'manifestJson', 'httpPort', 'location', 'dnsRecordId',
+    'accessRestriction', 'lastBackupId', 'lastBackupConfigJson', 'oldConfigJson' ].join(',');
 
-var PORT_BINDINGS_FIELDS = [ 'hostPort', 'containerPort', 'appId' ].join(',');
+var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
+    'apps.accessRestriction', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson' ].join(',');
+
+var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
 function postProcess(result) {
-    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
+    assert.strictEqual(typeof result, 'object');
 
+    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
+    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
+    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
+    delete result.lastBackupConfigJson;
+
+    assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
+    result.oldConfig = safe.JSON.parse(result.oldConfigJson);
+    delete result.oldConfigJson;
+
     assert(result.hostPorts === null || typeof result.hostPorts === 'string');
-    assert(result.containerPorts === null || typeof result.containerPorts === 'string');
+    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
 
     result.portBindings = { };
     var hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
-    var containerPorts = result.containerPorts === null ? [ ] : result.containerPorts.split(',');
+    var environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
 
     delete result.hostPorts;
-    delete result.containerPorts;
+    delete result.environmentVariables;
 
-    for (var i = 0; i < hostPorts.length; i++) {
-        result.portBindings[containerPorts[i]] = hostPorts[i];
+    for (var i = 0; i < environmentVariables.length; i++) {
+        result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
     }
 }
 
 function get(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
-        + ' FROM apps LEFT OUTER JOIN appPortBindings WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        postProcess(result[0]);
 
-        postProcess(result);
-
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
 function getBySubdomain(subdomain, callback) {
-    assert(typeof subdomain === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        postProcess(result[0]);
 
-        postProcess(result);
-
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
 function getByHttpPort(httpPort, callback) {
-    assert(typeof httpPort === 'number');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof httpPort, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        postProcess(result[0]);
 
-        postProcess(result);
-
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
 function getAll(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
         + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
         + ' GROUP BY apps.id ORDER BY apps.id', function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        if (typeof results === 'undefined') results = [ ];
-
         results.forEach(postProcess);
 
         callback(null, results);
     });
 }
 
-function add(id, appStoreId, location, portBindings, accessRestriction, callback) {
-    assert(typeof id === 'string');
-    assert(typeof appStoreId === 'string');
-    assert(typeof location === 'string');
-    assert(typeof portBindings === 'object');
-    assert(typeof accessRestriction === 'string');
-    assert(typeof callback === 'function');
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof manifest.version, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
-    var conn = database.beginTransaction();
+    var manifestJson = JSON.stringify(manifest);
 
-    conn.run('INSERT INTO apps (id, appStoreId, installationState, location, accessRestriction) VALUES (?, ?, ?, ?, ?)',
-           [ id, appStoreId, exports.ISTATE_PENDING_INSTALL, location, accessRestriction ], function (error) {
-        if (error || !this.lastID) database.rollback(conn);
+    var queries = [ ];
+    queries.push({
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestriction) VALUES (?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestriction ]
+    });
 
-        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-
-        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        async.eachSeries(Object.keys(portBindings), function iterator(containerPort, callback) {
-            conn.run('INSERT INTO appPortBindings (hostPort, containerPort, appId) VALUES (?, ?, ?)',
-                     [ portBindings[containerPort], containerPort, id ], callback);
-        }, function done(error) {
-            if (error) database.rollback(conn);
-
-            if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-
-            if (error /* || !this.lastID*/) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-            database.commit(conn, callback);
+    Object.keys(portBindings).forEach(function (env) {
+        queries.push({
+            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, appId) VALUES (?, ?, ?)',
+            args: [ env, portBindings[env], id ]
         });
     });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
 }
 
 function exists(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
+    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null, typeof result !== 'undefined');
+        return callback(null, result.length !== 0);
     });
 }
 
 function getPortBindings(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
+    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        results = results || [ ];
         var portBindings = { };
         for (var i = 0; i < results.length; i++) {
-            portBindings[results[i].containerPort] = results[i].hostPort;
+            portBindings[results[i].environmentVariable] = results[i].hostPort;
         }
 
         callback(null, portBindings);
@@ -211,29 +222,29 @@ function getPortBindings(id, callback) {
 }
 
 function del(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    var conn = database.beginTransaction();
-    conn.run('DELETE FROM appPortBindings WHERE appId = ?', [ id ], function (error) {
-        conn.run('DELETE FROM apps WHERE id = ?', [ id ], function (error) {
-            if (error || this.changes !== 1) database.rollback(conn);
+    var queries = [
+        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
+    ];
 
-            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-            if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    database.transaction(queries, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-            database.commit(conn, callback);
-        });
+        callback(null);
     });
 }
 
 function clear(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     async.series([
-        database.run.bind(null, 'DELETE FROM appPortBindings'),
-        database.run.bind(null, 'DELETE FROM apps'),
-        database.run.bind(null, 'DELETE FROM appAddonConfigs')
+        database.query.bind(null, 'DELETE FROM appPortBindings'),
+        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
+        database.query.bind(null, 'DELETE FROM apps')
     ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         return callback(null);
@@ -241,150 +252,154 @@ function clear(callback) {
 }
 
 function update(id, app, callback) {
-    updateWithConstraints(id, app, callback);
+    updateWithConstraints(id, app, '', callback);
 }
 
 function updateWithConstraints(id, app, constraints, callback) {
-    assert(typeof id === 'string');
-    assert(typeof app === 'object');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof constraints, 'string');
+    assert.strictEqual(typeof callback, 'function');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
 
-    if (typeof constraints === 'function') {
-        callback = constraints;
-        constraints = '';
-    } else {
-        assert(typeof constraints === 'string');
-        assert(typeof callback === 'function');
-    }
+    var queries = [ ];
 
-    var portBindings = app.portBindings || { };
-
-    var conn = database.beginTransaction();
-    async.eachSeries(Object.keys(portBindings), function iterator(containerPort, callback) {
-        var values = [ portBindings[containerPort], containerPort, id ];
-        conn.run('UPDATE appPortBindings SET hostPort = ? WHERE containerPort = ? AND appId = ?', values, callback);
-    }, function seriesDone(error) {
-        if (error) {
-            database.rollback(conn);
-            return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        }
-
-        var args = [ ], values = [ ];
-        for (var p in app) {
-            if (!app.hasOwnProperty(p)) continue;
-
-            if (p === 'manifest') {
-                args.push('manifestJson = ?');
-                values.push(JSON.stringify(app[p]));
-            } else if (p !== 'portBindings') {
-                args.push(p + ' = ?');
-                values.push(app[p]);
-            }
-        }
-
-        if (values.length === 0) return database.commit(conn, callback);
-
-        values.push(id);
-
-        conn.run('UPDATE apps SET ' + args.join(', ') + ' WHERE id = ? ' + constraints, values, function (error) {
-            if (error || this.changes !== 1) database.rollback(conn);
-            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-            if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-            database.commit(conn, callback);
+    if ('portBindings' in app) {
+        var portBindings = app.portBindings || { };
+        // replace entries by app id
+        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
+        Object.keys(portBindings).forEach(function (env) {
+            var values = [ portBindings[env], env, id ];
+            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, environmentVariable, appId) VALUES(?, ?, ?)', args: values });
         });
-    });
-}
-
-// sets health on installed apps that have a runState which is not null or pending
-function setHealth(appId, healthy, runState, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof healthy === 'boolean');
-    assert(typeof runState === 'string');
-    assert(typeof callback === 'function');
-
-    var values = {
-        healthy: healthy,
-        runState: runState
-    };
-
-    var constraints = 'AND runState NOT GLOB "pending_*" AND installationState = "installed"';
-    if (runState === exports.RSTATE_DEAD) { // don't mark stopped apps as dead
-        constraints += ' AND runState != "stopped"';
     }
 
-    updateWithConstraints(appId, values, constraints, callback);
-}
-
-function setInstallationCommand(appId, installationState, values, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof installationState === 'string');
-
-    if (typeof values === 'function') {
-        callback = values;
-        values = { };
-    } else {
-        assert(typeof values === 'object');
-        assert(typeof callback === 'function');
+    var fields = [ ], values = [ ];
+    for (var p in app) {
+        if (p === 'manifest') {
+            fields.push('manifestJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'lastBackupConfig') {
+            fields.push('lastBackupConfigJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'oldConfig') {
+            fields.push('oldConfigJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p !== 'portBindings') {
+            fields.push(p + ' = ?');
+            values.push(app[p]);
+        }
     }
 
-    values.installationState = installationState;
-
-    if (installationState === exports.ISTATE_PENDING_UNINSTALL) {
-        updateWithConstraints(appId, values, '', callback);
-    } else {
-        updateWithConstraints(appId, values, 'AND installationState NOT GLOB "pending_*"', callback);
+    if (values.length !== 0) {
+        values.push(id);
+        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
     }
-}
 
-function setRunCommand(appId, runState, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof runState === 'string');
-    assert(typeof callback === 'function');
-
-    var values = { runState: runState };
-    updateWithConstraints(appId, values, 'AND runState NOT GLOB "pending_*" AND installationState = "installed"', callback);
-}
-
-function getAppVersions(callback) {
-    assert(typeof callback === 'function');
-
-    database.all('SELECT id, appStoreId, version FROM apps', function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        results = results || [ ];
-        callback(null, results);
-    });
-}
-
-function setAddonConfig(appId, addonId, env, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof addonId === 'string');
-    assert(util.isArray(env));
-    assert(typeof callback === 'function');
-
-    if (env.length === 0) return callback(null);
-
-    var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
-    var args = [ ], queryArgs = [ ];
-    for (var i = 0; i < env.length; i++) {
-        args.push(appId, addonId, env[i]);
-        queryArgs.push('(?, ?, ?)');
-    }
-
-    database.run(query + queryArgs.join(','), args, function (error) {
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results[results.length - 1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         return callback(null);
     });
 }
 
-function unsetAddonConfig(appId, addonId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof addonId === 'string');
-    assert(typeof callback === 'function');
+// not sure if health should influence runState
+function setHealth(appId, health, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
+    var values = { health: health };
+
+    var constraints = 'AND runState NOT LIKE "pending_%" AND installationState = "installed"';
+
+    updateWithConstraints(appId, values, constraints, callback);
+}
+
+function setInstallationCommand(appId, installationState, values, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof installationState, 'string');
+
+    if (typeof values === 'function') {
+        callback = values;
+        values = { };
+    } else {
+        assert.strictEqual(typeof values, 'object');
+        assert.strictEqual(typeof callback, 'function');
+    }
+
+    values.installationState = installationState;
+    values.installationProgress = '';
+
+    // Rules are:
+    // uninstall is allowed in any state
+    // restore is allowed from installed or error state
+    // update and configure are allowed only in installed state
+
+    if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
+        updateWithConstraints(appId, values, '', callback);
+    } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
+        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
+        updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else {
+        callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
+    }
+}
+
+function setRunCommand(appId, runState, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof runState, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var values = { runState: runState };
+    updateWithConstraints(appId, values, 'AND runState NOT LIKE "pending_%" AND installationState = "installed"', callback);
+}
+
+function getAppStoreIds(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAddonConfig(appId, addonId, env, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(util.isArray(env));
+    assert.strictEqual(typeof callback, 'function');
+
+    unsetAddonConfig(appId, addonId, function (error) {
+        if (error) return callback(error);
+
+        if (env.length === 0) return callback(null);
+
+        var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
+        var args = [ ], queryArgs = [ ];
+        for (var i = 0; i < env.length; i++) {
+            args.push(appId, addonId, env[i]);
+            queryArgs.push('(?, ?, ?)');
+        }
+
+        database.query(query + queryArgs.join(','), args, function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            return callback(null);
+        });
+    });
+}
+
+function unsetAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -392,10 +407,10 @@ function unsetAddonConfig(appId, addonId, callback) {
 }
 
 function unsetAddonConfigByAppId(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -403,29 +418,29 @@ function unsetAddonConfigByAppId(appId, callback) {
 }
 
 function getAddonConfig(appId, addonId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof addonId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, result) {
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         var config = [ ];
-        result.forEach(function (v) { config.push(v.value); });
+        results.forEach(function (v) { config.push(v.value); });
 
         callback(null, config);
     });
 }
 
 function getAddonConfigByAppId(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, result) {
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         var config = [ ];
-        result.forEach(function (v) { config.push(v.value); });
+        results.forEach(function (v) { config.push(v.value); });
 
         callback(null, config);
     });

src/apps.js

@@ -2,103 +2,92 @@
 
 'use strict';
 
-var appdb = require('./appdb.js'),
-    assert = require('assert'),
-    child_process = require('child_process'),
-    config = require('../config.js'),
-    DatabaseError = require('./databaseerror.js'),
-    debug = require('debug')('box:apps'),
-    docker = require('./docker.js'),
-    fs = require('fs'),
-    os = require('os'),
-    paths = require('./paths.js'),
-    split = require('split'),
-    stream = require('stream'),
-    util = require('util');
-
 exports = module.exports = {
     AppsError: AppsError,
 
-    initialize: initialize,
-    uninitialize: uninitialize,
     get: get,
     getBySubdomain: getBySubdomain,
     getAll: getAll,
+    purchase: purchase,
     install: install,
     configure: configure,
     uninstall: uninstall,
+
+    restore: restore,
+    restoreApp: restoreApp,
+
     update: update,
 
+    backup: backup,
+    backupApp: backupApp,
+
     getLogStream: getLogStream,
     getLogs: getLogs,
 
     start: start,
     stop: stop,
 
+    exec: exec,
+
+    checkManifestConstraints: checkManifestConstraints,
+
+    setRestorePoint: setRestorePoint,
+
+    autoupdateApps: autoupdateApps,
+
     // exported for testing
     _validateHostname: validateHostname,
     _validatePortBindings: validatePortBindings
 };
 
-var gTasks = { };
+var addons = require('./addons.js'),
+    appdb = require('./appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    backups = require('./backups.js'),
+    BackupsError = require('./backups.js').BackupsError,
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apps'),
+    docker = require('./docker.js'),
+    fs = require('fs'),
+    manifestFormat = require('cloudron-manifestformat'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    shell = require('./shell.js'),
+    split = require('split'),
+    superagent = require('superagent'),
+    taskmanager = require('./taskmanager.js'),
+    util = require('util'),
+    validator = require('validator');
 
-function initialize(callback) {
-    assert(typeof callback === 'function');
+var BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
+    RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
+    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
 
-    resume(callback); // TODO: potential race here since resume is async
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-function startTask(appId) {
-    assert(typeof appId === 'string');
-    assert(!(appId in gTasks));
-
-    gTasks[appId] = child_process.fork(__dirname + '/apptask.js', [ appId ]);
-    gTasks[appId].once('exit', function (code, signal) {
-        debug('Task completed :' + appId);
-        delete gTasks[appId];
-    });
-}
-
-function stopTask(appId) {
-    assert(typeof appId === 'string');
-
-    if (gTasks[appId]) {
-        debug('Killing existing task : ' + gTasks[appId].pid);
-        gTasks[appId].kill();
-        delete gTasks[appId];
-    }
-}
-
-// resume install and uninstalls
-function resume(callback) {
-    assert(typeof callback === 'function');
-
-    appdb.getAll(function (error, apps) {
-        if (error) return callback(error);
-
-        apps.forEach(function (app) {
-            debug('Creating process for ' + app.id + ' with state ' + app.installationState);
-            startTask(app.id);
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+            callback();
         });
-
-        callback(null);
-    });
-}
-
-function uninitialize(callback) {
-    assert(typeof callback === 'function');
-
-    for (var appId in gTasks) {
-        stopTask(appId);
-    }
-
-    callback(null);
+    };
 }
 
 // http://dustinsenos.com/articles/customErrorsInNode
 // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
 function AppsError(reason, errorOrMessage) {
-    assert(typeof reason === 'string');
+    assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
 
     Error.call(this);
@@ -117,21 +106,27 @@ function AppsError(reason, errorOrMessage) {
 }
 util.inherits(AppsError, Error);
 AppsError.INTERNAL_ERROR = 'Internal Error';
+AppsError.EXTERNAL_ERROR = 'External Error';
 AppsError.ALREADY_EXISTS = 'Already Exists';
 AppsError.NOT_FOUND = 'Not Found';
 AppsError.BAD_FIELD = 'Bad Field';
 AppsError.BAD_STATE = 'Bad State';
+AppsError.PORT_RESERVED = 'Port Reserved';
+AppsError.PORT_CONFLICT = 'Port Conflict';
+AppsError.BILLING_REQUIRED = 'Billing Required';
 
 // Hostname validation comes from RFC 1123 (section 2.1)
 // Domain name validation comes from RFC 2181 (Name syntax)
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name
 function validateHostname(location, fqdn) {
-    var RESERVED_LOCATIONS = [ 'admin' ];
+    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION ];
 
     if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new Error(location + ' is reserved');
 
-    if ((location.length + 1 + /* hyphen */ + fqdn.indexOf('.')) > 63) return new Error('Hostname length cannot be greater than 63');
+    if (location === '') return null; // bare location
+
+    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new Error('Hostname length cannot be greater than 63');
     if (location.match(/^[A-Za-z0-9-]+$/) === null) return new Error('Hostname can only contain alphanumerics and hyphen');
     if (location[0] === '-' || location[location.length-1] === '-') return new Error('Hostname cannot start or end with hyphen');
     if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new Error('FQDN length exceeds 253 characters');
@@ -140,52 +135,83 @@ function validateHostname(location, fqdn) {
 }
 
 // validate the port bindings
-function validatePortBindings(portBindings) {
+function validatePortBindings(portBindings, tcpPorts) {
     // keep the public ports in sync with firewall rules in scripts/initializeBaseUbuntuImage.sh
+    // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
+    // for custom tcp ports
     var RESERVED_PORTS = [
         22, /* ssh */
         25, /* smtp */
         53, /* dns */
         80, /* http */
         443, /* https */
-        2003, /* graphite */
-        2004, /* graphite */
+        2003, /* graphite (lo) */
+        2004, /* graphite (lo) */
         2020, /* install server */
-        3000, /* app server */
-        8000 /* graphite */
+        config.get('port'), /* app server (lo) */
+        config.get('internalPort'), /* internal app server (lo) */
+        3306, /* mysql (lo) */
+        8000 /* graphite (lo) */
     ];
 
-    for (var containerPort in portBindings) {
-        var containerPortInt = parseInt(containerPort, 10);
-        if (isNaN(containerPortInt) || containerPortInt <= 0 || containerPortInt > 65535) {
-            return new Error(containerPort + ' is not a valid port');
-        }
+    if (!portBindings) return null;
 
-        var hostPortInt = parseInt(portBindings[containerPort], 10);
-        if (isNaN(hostPortInt) || hostPortInt <= 1024 || hostPortInt > 65535) {
-            return new Error(portBindings[containerPort] + ' is not a valid port');
-        }
+    var env;
+    for (env in portBindings) {
+        if (!/^[a-zA-Z0-9_]+$/.test(env)) return new AppsError(AppsError.BAD_FIELD, env + ' is not valid environment variable');
 
-        if (RESERVED_PORTS.indexOf(hostPortInt) !== -1) return new Error(hostPortInt + ' is reserved');
+        if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
+        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
+
+        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, + portBindings[env]);
+    }
+
+    // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
+    // that the user wants the service disabled
+    tcpPorts = tcpPorts || { };
+    for (env in portBindings) {
+        if (!(env in tcpPorts)) return new AppsError(AppsError.BAD_FIELD, 'Invalid portBindings ' + env);
     }
 
     return null;
 }
 
-function getIconURLSync(app) {
+function getDuplicateErrorDetails(location, portBindings, error) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(error.reason, DatabaseError.ALREADY_EXISTS);
+
+    var match = error.message.match(/ER_DUP_ENTRY: Duplicate entry '(.*)' for key/);
+    if (!match) {
+        console.error('Unexpected SQL error message.', error);
+        return new AppsError(AppsError.INTERNAL_ERROR);
+    }
+
+    // check if the location conflicts
+    if (match[1] === location) return new AppsError(AppsError.ALREADY_EXISTS);
+
+    // check if any of the port bindings conflict
+    for (var env in portBindings) {
+        if (portBindings[env] === parseInt(match[1])) return new AppsError(AppsError.PORT_CONFLICT, match[1]);
+    }
+
+    return new AppsError(AppsError.ALREADY_EXISTS);
+}
+
+function getIconUrlSync(app) {
     var iconPath = paths.APPICONS_DIR + '/' + app.id + '.png';
     return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
 }
 
 function get(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        app.icon = getIconURLSync(app);
+        app.iconUrl = getIconUrlSync(app);
         app.fqdn = config.appFqdn(app.location);
 
         callback(null, app);
@@ -193,14 +219,14 @@ function get(appId, callback) {
 }
 
 function getBySubdomain(subdomain, callback) {
-    assert(typeof subdomain === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     appdb.getBySubdomain(subdomain, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        app.icon = getIconURLSync(app);
+        app.iconUrl = getIconUrlSync(app);
         app.fqdn = config.appFqdn(app.location);
 
         callback(null, app);
@@ -208,13 +234,13 @@ function getBySubdomain(subdomain, callback) {
 }
 
 function getAll(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     appdb.getAll(function (error, apps) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
         apps.forEach(function (app) {
-            app.icon = getIconURLSync(app);
+            app.iconUrl = getIconUrlSync(app);
             app.fqdn = config.appFqdn(app.location);
         });
 
@@ -234,111 +260,196 @@ function validateAccessRestriction(accessRestriction) {
     }
 }
 
-function install(appId, appStoreId, username, password, location, portBindings, accessRestriction, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof username === 'string');
-    assert(typeof password === 'string');
-    assert(typeof location === 'string');
-    assert(!portBindings || typeof portBindings === 'object');
-    assert(typeof accessRestriction === 'string');
-    assert(typeof callback === 'function');
+function purchase(appStoreId, callback) {
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    var error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+    // Skip purchase if appStoreId is empty
+    if (appStoreId === '') return callback(null);
 
-    error = validatePortBindings(portBindings);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+    var url = config.apiServerOrigin() + '/api/v1/apps/' + appStoreId + '/purchase';
 
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    debug('Will install app with id : ' + appId);
-
-    appdb.add(appId, appStoreId, location.toLowerCase(), portBindings, accessRestriction, function (error) {
-        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new AppsError(AppsError.ALREADY_EXISTS));
+    superagent.post(url).query({ token: config.token() }).end(function (error, res) {
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        stopTask(appId);
-        startTask(appId);
+        if (res.status === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+        if (res.status !== 201 && res.status !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
 
         callback(null);
     });
 }
 
-function configure(appId, username, password, location, portBindings, accessRestriction, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof username === 'string');
-    assert(typeof password === 'string');
-    assert(!portBindings || typeof portBindings === 'object');
-    assert(typeof accessRestriction === 'string');
-    assert(typeof callback === 'function');
+function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'string');
+    assert(!icon || typeof icon === 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    var error = location ? validateHostname(location, config.fqdn()) : null;
+    var error = manifestFormat.parse(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
+
+    error = checkManifestConstraints(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+
+    error = validateHostname(location, config.fqdn());
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
-    error = portBindings ? validatePortBindings(portBindings) : null;
+    error = validatePortBindings(portBindings, manifest.tcpPorts);
     if (error) return callback(error);
 
     error = validateAccessRestriction(accessRestriction);
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
-    var values = { };
-    if (location) values.location = location.toLowerCase();
-    values.portBindings = portBindings;
-    values.accessRestriction = accessRestriction;
+    if (icon) {
+        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
 
-    debug('Will install app with id:%s', appId);
+        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+        }
+    }
 
-    appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_CONFIGURE, values, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+    debug('Will install app with id : ' + appId);
 
-        stopTask(appId);
-        startTask(appId);
+    purchase(appStoreId, function (error) {
+        if (error) return callback(error);
 
-        callback(null);
+        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
     });
 }
 
-function update(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+function configure(appId, location, portBindings, accessRestriction, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateHostname(location, config.fqdn());
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    error = validateAccessRestriction(accessRestriction);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        error = validatePortBindings(portBindings, app.manifest.tcpPorts);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+        var values = {
+            location: location.toLowerCase(),
+            accessRestriction: accessRestriction,
+            portBindings: portBindings,
+
+            oldConfig: {
+                location: app.location,
+                accessRestriction: app.accessRestriction,
+                portBindings: app.portBindings
+            }
+        };
+
+        debug('Will configure app with id:%s values:%j', appId, values);
+
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_CONFIGURE, values, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function update(appId, force, manifest, portBindings, icon, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof force, 'boolean');
+    assert(manifest && typeof manifest === 'object');
+    assert(!portBindings || typeof portBindings === 'object');
+    assert(!icon || typeof icon === 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Will update app with id:%s', appId);
 
-    appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UPDATE, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+    var error = manifestFormat.parse(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
+
+    error = checkManifestConstraints(manifest);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed:' + error.message));
+
+    error = validatePortBindings(portBindings, manifest.tcpPorts);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    if (icon) {
+        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+
+        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+        }
+    }
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        stopTask(appId);
-        startTask(appId);
+        var values = {
+            manifest: manifest,
+            portBindings: portBindings,
+            oldConfig: {
+                manifest: app.manifest,
+                portBindings: app.portBindings
+            }
+        };
 
-        callback(null);
+        appdb.setInstallationCommand(appId, force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, portBindings, error));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
     });
 }
 
-function getLogStream(appId, options, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof options === 'object');
-    assert(typeof callback === 'function');
+function getLogStream(appId, fromLine, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof fromLine, 'number'); // behaves like tail -n
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Getting logs for %s', appId);
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, 'App not installed'));
+        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
 
         var container = docker.getContainer(app.containerId);
+        var tail = fromLine < 0 ? -fromLine : 'all';
+
         // note: cannot access docker file directly because it needs root access
-        container.logs({ stdout: true, stderr: true, follow: true, timestamps: true, tail: 'all' }, function (error, logStream) {
+        container.logs({ stdout: true, stderr: true, follow: true, timestamps: true, tail: tail }, function (error, logStream) {
             if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             var lineCount = 0;
             var skipLinesStream = split(function mapper(line) {
-                if (++lineCount < options.fromLine) return undefined;
-                return JSON.stringify({ lineNumber: lineCount, log: line });
+                if (++lineCount < fromLine) return undefined;
+                var timestamp = line.substr(0, line.indexOf(' ')); // sometimes this has square brackets around it
+                return JSON.stringify({ lineNumber: lineCount, timestamp: timestamp.replace(/[[\]]/g,''), log: line.substr(timestamp.length + 1) });
             });
             skipLinesStream.close = logStream.req.abort;
             logStream.pipe(skipLinesStream);
@@ -348,15 +459,15 @@ function getLogStream(appId, options, callback) {
 }
 
 function getLogs(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Getting logs for %s', appId);
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, 'App not installed'));
+        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
 
         var container = docker.getContainer(app.containerId);
         // note: cannot access docker file directly because it needs root access
@@ -369,9 +480,53 @@ function getLogs(appId, callback) {
     });
 }
 
+function restore(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will restore app with id:%s', appId);
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        var restoreConfig = app.lastBackupConfig;
+        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+
+        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
+        error = checkManifestConstraints(restoreConfig.manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+
+        error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
+        if (error) return callback(error);
+
+        // ## should probably query new location, access restriction from user
+        var values = {
+            manifest: restoreConfig.manifest,
+            portBindings: restoreConfig.portBindings,
+
+            oldConfig: {
+                location: app.location,
+                accessRestriction: app.accessRestriction,
+                portBindings: app.portBindings,
+                manifest: app.manifest
+            }
+        };
+
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
 function uninstall(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Will uninstall app with id:%s', appId);
 
@@ -379,16 +534,15 @@ function uninstall(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        stopTask(appId);
-        startTask(appId);
+        taskmanager.restartAppTask(appId); // since uninstall is allowed from any state, kill current task
 
         callback(null);
     });
 }
 
 function start(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Will start app with id:%s', appId);
 
@@ -396,16 +550,15 @@ function start(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        stopTask(appId);
-        startTask(appId);
+        taskmanager.restartAppTask(appId);
 
         callback(null);
     });
 }
 
 function stop(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Will stop app with id:%s', appId);
 
@@ -413,10 +566,201 @@ function stop(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        stopTask(appId);
-        startTask(appId);
+        taskmanager.restartAppTask(appId);
 
         callback(null);
     });
 }
 
+function checkManifestConstraints(manifest) {
+    if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
+        return new Error('Box version exceeds Apps maxBoxVersion');
+    }
+
+    if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, config.version())) {
+        return new Error('minBoxVersion exceeds Box version');
+    }
+
+    return null;
+}
+
+function exec(appId, options, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert(options && typeof options === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var cmd = options.cmd || [ '/bin/bash' ];
+    assert(util.isArray(cmd) && cmd.length > 0);
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        var container = docker.getContainer(app.containerId);
+
+       var execOptions = {
+            AttachStdin: true,
+            AttachStdout: true,
+            AttachStderr: true,
+            Tty: true,
+            Cmd: cmd
+        };
+
+        container.exec(execOptions, function (error, exec) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+            var startOptions = {
+                Detach: false,
+                Tty: true,
+                stdin: true // this is a dockerode option that enabled openStdin in the modem
+            };
+            exec.start(startOptions, function(error, stream) {
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                if (options.rows && options.columns) {
+                    exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); });
+                }
+
+                return callback(null, stream);
+            });
+        });
+    });
+}
+
+function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lastBackupId, 'string');
+    assert.strictEqual(typeof lastBackupConfig, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
+    assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    function canAutoupdateApp(app, newManifest) {
+        // TODO: maybe check the description as well?
+        if (!newManifest.tcpPorts && !app.portBindings) return true;
+        if (!newManifest.tcpPorts || !app.portBindings) return false;
+
+        for (var env in newManifest.tcpPorts) {
+            if (!(env in app.portBindings)) return false;
+       }
+
+        return true;
+    }
+
+    if (!updateInfo) return callback(null);
+
+    async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
+        get(appId, function (error, app) {
+            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+                return iteratorDone();
+            }
+
+           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+                if (error) debug('Error initiating autoupdate of %s', appId);
+
+                iteratorDone(null);
+            });
+        });
+    }, callback);
+}
+
+function backupApp(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    function canBackupApp(app) {
+        // only backup apps that are installed or pending configure. Rest of them are in some
+        // state not good for consistent backup (i.e addons may not have been setup completely)
+        return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+                app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+                app.installationState === appdb.ISTATE_PENDING_BACKUP ||
+                app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+    }
+
+    if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy'));
+
+    var appConfig = {
+        manifest: app.manifest,
+        location: app.location,
+        portBindings: app.portBindings,
+        accessRestriction: app.accessRestriction
+    };
+
+    if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
+        return callback(safe.error);
+    }
+
+    backups.getBackupUrl(app, null, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id);
+
+        async.series([
+            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
+            addons.backupAddons.bind(null, app, addonsToBackup),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
+            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
+        ], function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            debugApp(app, 'backupApp: successful id:%s', result.id);
+
+            setRestorePoint(app.id, result.id, appConfig, function (error) {
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                return callback(null, result.id);
+            });
+        });
+    });
+}
+
+function backup(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(appId, function (error, app) {
+        if (error && error.reason === AppsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_BACKUP, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            taskmanager.restartAppTask(appId);
+
+            callback(null);
+        });
+    });
+}
+
+function restoreApp(app, addonsToRestore, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof callback, 'function');
+    assert(app.lastBackupId);
+
+    backups.getRestoreUrl(app.lastBackupId, function (error, result) {
+        if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
+
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            addons.restoreAddons(app, addonsToRestore, callback);
+        });
+    });
+}
+

src/auth.js

@@ -2,12 +2,16 @@
 
 'use strict';
 
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize
+};
+
 var assert = require('assert'),
     BasicStrategy = require('passport-http').BasicStrategy,
     BearerStrategy = require('passport-http-bearer').Strategy,
     clientdb = require('./clientdb'),
     ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
-    database = require('./database'),
     DatabaseError = require('./databaseerror'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
@@ -16,15 +20,11 @@ var assert = require('assert'),
     tokendb = require('./tokendb'),
     user = require('./user'),
     userdb = require('./userdb'),
-    UserError = user.UserError;
-
-exports = module.exports = {
-    initialize: initialize,
-    uninitialize: uninitialize
-};
+    UserError = user.UserError,
+    _ = require('underscore');
 
 function initialize(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     passport.serializeUser(function (user, callback) {
         callback(null, user.username);
@@ -42,21 +42,31 @@ function initialize(callback) {
     });
 
     passport.use(new LocalStrategy(function (username, password, callback) {
-        user.verify(username, password, function (error, result) {
-            if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
-            if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
-            if (error) return callback(error);
-            if (!result) return callback(null, false);
-            callback(null, database.removePrivates(result));
-        });
+        if (username.indexOf('@') === -1) {
+            user.verify(username, password, function (error, result) {
+                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, _.pick(result, 'id', 'username', 'email', 'admin'));
+            });
+        } else {
+            user.verifyWithEmail(username, password, function (error, result) {
+                if (error && error.reason === UserError.NOT_FOUND) return callback(null, false);
+                if (error && error.reason === UserError.WRONG_PASSWORD) return callback(null, false);
+                if (error) return callback(error);
+                if (!result) return callback(null, false);
+                callback(null, _.pick(result, 'id', 'username', 'email', 'admin'));
+            });
+        }
     }));
 
     passport.use(new BasicStrategy(function (username, password, callback) {
         if (username.indexOf('cid-') === 0) {
-            debug('BasicStrategy: detected clientId %s instead of username:password', username);
+            debug('BasicStrategy: detected client id %s instead of username:password', username);
             // username is actually client id here
             // password is client secret
-            clientdb.getByClientId(username, function (error, client) {
+            clientdb.get(username, function (error, client) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
                 if (client.clientSecret != password) return callback(null, false);
@@ -74,7 +84,7 @@ function initialize(callback) {
     }));
 
     passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.getByClientId(clientId, function(error, client) {
+        clientdb.get(clientId, function(error, client) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
             if (error) { return callback(error); }
             if (client.clientSecret != clientSecret) { return callback(null, false); }
@@ -87,13 +97,32 @@ function initialize(callback) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
-            userdb.get(token.userId, function (error, user) {
+            // scopes here can define what capabilities that token carries
+            // passport put the 'info' object into req.authInfo, where we can further validate the scopes
+            var info = { scope: token.scope };
+            var tokenType;
+
+            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
+                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
+                tokenType = tokendb.TYPE_DEV;
+            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
+                tokenType = tokendb.TYPE_APP;
+                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
+            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
+                tokenType = tokendb.TYPE_USER;
+                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
+            } else {
+                // legacy tokens assuming a user access token
+                tokenType = tokendb.TYPE_USER;
+            }
+
+            userdb.get(token.identifier, function (error, user) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
 
-                // scopes here can define what capabilities that token carries
-                // passport put the 'info' object into req.authInfo, where we can further validate the scopes
-                var info = { scope: token.scope };
+                // amend the tokenType of the token owner
+                user.tokenType = tokenType;
+
                 callback(null, user, info);
             });
         });
@@ -103,7 +132,7 @@ function initialize(callback) {
 }
 
 function uninitialize(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     callback(null);
 }

src/authcodedb.js

@@ -2,64 +2,74 @@
 
 'use strict';
 
-var assert = require('assert'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror'),
-    debug = require('debug')('box:authcodedb');
-
 exports = module.exports = {
     get: get,
     add: add,
     del: del,
-    clear: clear
+    delExpired: delExpired,
+
+    _clear: clear
 };
 
-var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId' ].join(',');
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror');
+
+var AUTHCODES_FIELDS = [ 'authCode', 'userId', 'clientId', 'expiresAt' ].join(',');
 
 function get(authCode, callback) {
-    assert(typeof authCode === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof authCode, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
+    database.query('SELECT ' + AUTHCODES_FIELDS + ' FROM authcodes WHERE authCode = ? AND expiresAt > ?', [ authCode, Date.now() ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
-function add(authCode, clientId, userId, callback) {
-    assert(typeof authCode === 'string');
-    assert(typeof clientId === 'string');
-    assert(typeof userId === 'string');
-    assert(typeof callback === 'function');
+function add(authCode, clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof authCode, 'string');
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('INSERT INTO authcodes (authCode, clientId, userId) VALUES (?, ?, ?)',
-            [ authCode, clientId, userId ], function (error) {
-        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.query('INSERT INTO authcodes (authCode, clientId, userId, expiresAt) VALUES (?, ?, ?, ?)',
+            [ authCode, clientId, userId, expiresAt ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
     });
 }
 
 function del(authCode, callback) {
-    assert(typeof authCode === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof authCode, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error) {
+    database.query('DELETE FROM authcodes WHERE authCode = ?', [ authCode ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         callback(null);
     });
 }
 
+function delExpired(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM authcodes WHERE expiresAt <= ?', [ Date.now() ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        return callback(null, result.affectedRows);
+    });
+}
+
 function clear(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM authcodes', function (error) {
+    database.query('DELETE FROM authcodes', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);

src/backups.js

@@ -0,0 +1,95 @@
+'use strict';
+
+exports = module.exports = {
+    BackupsError: BackupsError,
+
+    getAllPaged: getAllPaged,
+
+    getBackupUrl: getBackupUrl,
+    getRestoreUrl: getRestoreUrl
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:backups'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function BackupsError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(BackupsError, Error);
+BackupsError.EXTERNAL_ERROR = 'external error';
+BackupsError.INTERNAL_ERROR = 'internal error';
+
+function getAllPaged(page, perPage, callback) {
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+
+    superagent.get(url).query({ token: config.token() }).end(function (error, result) {
+        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+        if (!result.body || !util.isArray(result.body.backups)) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+
+        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
+        return callback(null, result.body.backups);
+    });
+}
+
+function getBackupUrl(app, appBackupIds, callback) {
+    assert(!app || typeof app === 'object');
+    assert(!appBackupIds || util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+
+    var data = {
+        boxVersion: config.version(),
+        appId: app ? app.id : null,
+        appVersion: app ? app.manifest.version : null,
+        appBackupIds: appBackupIds
+    };
+
+    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
+        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+
+        return callback(null, result.body);
+    });
+}
+
+function getRestoreUrl(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+
+    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
+        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
+        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
+
+        return callback(null, result.body);
+    });
+}
+
+

src/clientdb.js

@@ -2,170 +2,135 @@
 
 'use strict';
 
-var assert = require('assert'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js'),
-    debug = require('debug')('box:clientdb');
-
 exports = module.exports = {
     get: get,
     getAll: getAll,
-    getAllWithDetails: getAllWithDetails,
-    getByClientId: getByClientId,
+    getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
-    replaceByAppId: replaceByAppId,
+    update: update,
     getByAppId: getByAppId,
     delByAppId: delByAppId,
-    clear: clear
+
+    _clear: clear
 };
 
-var CLIENTS_FIELDS = [ 'id', 'appId', 'clientId', 'clientSecret', 'name', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.clientId', 'clients.clientSecret', 'clients.name', 'clients.redirectURI', 'clients.scope' ].join(',');
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js');
+
+var CLIENTS_FIELDS = [ 'id', 'appId', 'clientSecret', 'redirectURI', 'scope' ].join(',');
+var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
 
 function get(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE id = ?', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
 function getAll(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT ' + CLIENTS_FIELDS + ' FROM clients', [ ], function (error, results) {
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients ORDER BY appId', function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (typeof results === 'undefined') results = [];
 
         callback(null, results);
     });
 }
 
-function getAllWithDetails(callback) {
-    assert(typeof callback === 'function');
+function getAllWithTokenCountByIdentifier(identifier, callback) {
+    assert.strictEqual(typeof identifier, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    // TODO should this be per user?
-    database.all('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (typeof results === 'undefined') results = [];
-
         callback(null, results);
-    });
-}
 
-function getByClientId(clientId, callback) {
-    assert(typeof clientId === 'string');
-    assert(typeof callback === 'function');
-
-    database.get('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE clientId = ? LIMIT 1', [ clientId ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        return callback(null, result);
     });
 }
 
 function getByAppId(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? LIMIT 1', [ appId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result);
+        return callback(null, result[0]);
     });
 }
 
-function add(id, appId, clientId, clientSecret, name, redirectURI, scope, callback) {
-    assert(typeof id === 'string');
-    assert(typeof appId === 'string');
-    assert(typeof clientId === 'string');
-    assert(typeof clientSecret === 'string');
-    assert(typeof name === 'string');
-    assert(typeof redirectURI === 'string');
-    assert(typeof scope === 'string');
-    assert(typeof callback === 'function');
+function add(id, appId, clientSecret, redirectURI, scope, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof clientSecret, 'string');
+    assert.strictEqual(typeof redirectURI, 'string');
+    assert.strictEqual(typeof scope, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    var data = {
-        $id: id,
-        $appId: appId,
-        $clientId: clientId,
-        $clientSecret: clientSecret,
-        $name: name,
-        $redirectURI: redirectURI,
-        $scope: scope
-    };
+    var data = [ id, appId, clientSecret, redirectURI, scope ];
 
-    database.run('INSERT INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES ($id, $appId, $clientId, $clientSecret, $name, $redirectURI, $scope)', data, function (error) {
-        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.query('INSERT INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?)', data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function update(id, appId, clientSecret, redirectURI, scope, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof clientSecret, 'string');
+    assert.strictEqual(typeof redirectURI, 'string');
+    assert.strictEqual(typeof scope, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [ appId, clientSecret, redirectURI, scope, id ];
+
+    database.query('UPDATE clients SET appId = ?, clientSecret = ?, redirectURI = ?, scope = ? WHERE id = ?', data, function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         callback(null);
     });
 }
 
 function del(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM clients WHERE id = ?', [ id ], function (error) {
+    database.query('DELETE FROM clients WHERE id = ?', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         callback(null);
     });
 }
 
 function delByAppId(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM clients WHERE appId=?', [ appId ], function (error) {
+    database.query('DELETE FROM clients WHERE appId=?', [ appId ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         return callback(null);
     });
 }
 
-function replaceByAppId(id, appId, clientId, clientSecret, name, redirectURI, scope, callback) {
-    assert(typeof id === 'string');
-    assert(typeof appId === 'string');
-    assert(typeof clientId === 'string');
-    assert(typeof clientSecret === 'string');
-    assert(typeof name === 'string');
-    assert(typeof redirectURI === 'string');
-    assert(typeof scope === 'string');
-    assert(typeof callback === 'function');
-
-    var data = {
-        $id: id,
-        $appId: appId,
-        $clientId: clientId,
-        $clientSecret: clientSecret,
-        $name: name,
-        $redirectURI: redirectURI,
-        $scope: scope
-    };
-
-    database.run('INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES ($id, $appId, $clientId, $clientSecret, $name, $redirectURI, $scope)', data, function (error) {
-        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        callback(null);
-    });
-}
-
 function clear(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         return callback(null);

src/clients.js

@@ -0,0 +1,226 @@
+'use strict';
+
+exports = module.exports = {
+    ClientsError: ClientsError,
+
+    add: add,
+    get: get,
+    update: update,
+    del: del,
+    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getClientTokensByUserId: getClientTokensByUserId,
+    delClientTokensByUserId: delClientTokensByUserId
+};
+
+var assert = require('assert'),
+    util = require('util'),
+    hat = require('hat'),
+    appdb = require('./appdb.js'),
+    tokendb = require('./tokendb.js'),
+    constants = require('./constants.js'),
+    async = require('async'),
+    clientdb = require('./clientdb.js'),
+    DatabaseError = require('./databaseerror.js'),
+    uuid = require('node-uuid');
+
+function ClientsError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(ClientsError, Error);
+ClientsError.INVALID_SCOPE = 'Invalid scope';
+
+function validateScope(scope) {
+    assert.strictEqual(typeof scope, 'string');
+
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
+    if (scope === '*') return null;
+
+    // TODO maybe validate all individual scopes if they exist
+
+    return null;
+}
+
+function add(appIdentifier, redirectURI, scope, callback) {
+    assert.strictEqual(typeof appIdentifier, 'string');
+    assert.strictEqual(typeof redirectURI, 'string');
+    assert.strictEqual(typeof scope, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateScope(scope);
+    if (error) return callback(error);
+
+    var id = 'cid-' + uuid.v4();
+    var clientSecret = hat(256);
+
+    clientdb.add(id, appIdentifier, clientSecret, redirectURI, scope, function (error) {
+        if (error) return callback(error);
+
+        var client = {
+            id: id,
+            appId: appIdentifier,
+            clientSecret: clientSecret,
+            redirectURI: redirectURI,
+            scope: scope
+        };
+
+        callback(null, client);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.get(id, function (error, result) {
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
+// we only allow appIdentifier and redirectURI to be updated
+function update(id, appIdentifier, redirectURI, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appIdentifier, 'string');
+    assert.strictEqual(typeof redirectURI, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.get(id, function (error, result) {
+        if (error) return callback(error);
+
+        clientdb.update(id, appIdentifier, result.clientSecret, redirectURI, result.scope, function (error, result) {
+            if (error) return callback(error);
+            callback(null, result);
+        });
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.del(id, function (error, result) {
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
+function getAllWithDetailsByUserId(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
+        if (error) return callback(error);
+
+        // We have several types of records here
+        //   1) webadmin has an app id of 'webadmin'
+        //   2) oauth proxy records are always the app id prefixed with 'proxy-'
+        //   3) addon oauth records for apps prefixed with 'addon-'
+        //   4) external app records prefixed with 'external-'
+        //   5) normal apps on the cloudron without a prefix
+
+        var tmp = [];
+        async.each(results, function (record, callback) {
+            if (record.appId === constants.ADMIN_CLIENT_ID) {
+                record.name = constants.ADMIN_NAME;
+                record.location = constants.ADMIN_LOCATION;
+                record.type = 'webadmin';
+
+                tmp.push(record);
+
+                return callback(null);
+            } else if (record.appId === constants.TEST_CLIENT_ID) {
+                record.name = constants.TEST_NAME;
+                record.location = constants.TEST_LOCATION;
+                record.type = 'test';
+
+                tmp.push(record);
+
+                return callback(null);
+            }
+
+            var appId = record.appId;
+            var type = 'app';
+
+            // Handle our different types of oauth clients
+            if (record.appId.indexOf('addon-') === 0) {
+                appId = record.appId.slice('addon-'.length);
+                type = 'addon';
+            } else if (record.appId.indexOf('proxy-') === 0) {
+                appId = record.appId.slice('proxy-'.length);
+                type = 'proxy';
+            }
+
+            appdb.get(appId, function (error, result) {
+                if (error) {
+                    console.error('Failed to get app details for oauth client', result, error);
+                    return callback(null);  // ignore error so we continue listing clients
+                }
+
+                record.name = result.manifest.title + (record.appId.indexOf('proxy-') === 0 ? 'OAuth Proxy' : '');
+                record.location = result.location;
+                record.type = type;
+
+                tmp.push(record);
+
+                callback(null);
+            });
+        }, function (error) {
+            if (error) return callback(error);
+            callback(null, tmp);
+        });
+    });
+}
+
+function getClientTokensByUserId(clientId, userId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) {
+            // this can mean either that there are no tokens or the clientId is actually unknown
+            clientdb.get(clientId, function (error/*, result*/) {
+                if (error) return callback(error);
+                callback(null, []);
+            });
+            return;
+        }
+        if (error) return callback(error);
+        callback(null, result || []);
+    });
+}
+
+function delClientTokensByUserId(clientId, userId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) {
+            // this can mean either that there are no tokens or the clientId is actually unknown
+            clientdb.get(clientId, function (error/*, result*/) {
+                if (error) return callback(error);
+                callback(null);
+            });
+            return;
+        }
+        if (error) return callback(error);
+        callback(null);
+    });
+}

src/cloudron.js

@@ -2,7 +2,6 @@
 
 'use strict';
 
-// intentionally placed here because of circular dep with updater
 exports = module.exports = {
     CloudronError: CloudronError,
 
@@ -11,46 +10,72 @@ exports = module.exports = {
     activate: activate,
     getConfig: getConfig,
     getStatus: getStatus,
-    backup: backup,
 
-    getBackupUrl: getBackupUrl,
     setCertificate: setCertificate,
 
-    getIp: getIp
-};
+    sendHeartbeat: sendHeartbeat,
 
-var assert = require('assert'),
-    config = require('../config.js'),
-    debug = require('debug')('box:cloudron'),
+    update: update,
+    reboot: reboot,
+    migrate: migrate,
+    backup: backup,
+    ensureBackup: ensureBackup};
+
+var apps = require('./apps.js'),
+    AppsError = require('./apps.js').AppsError,
+    assert = require('assert'),
+    async = require('async'),
+    backups = require('./backups.js'),
+    BackupsError = require('./backups.js').BackupsError,
     clientdb = require('./clientdb.js'),
-    execFile = require('child_process').execFile,
+    config = require('./config.js'),
+    debug = require('debug')('box:cloudron'),
     fs = require('fs'),
-    os = require('os'),
+    locker = require('./locker.js'),
     path = require('path'),
     paths = require('./paths.js'),
+    progress = require('./progress.js'),
     safe = require('safetydance'),
+    settings = require('./settings.js'),
+    SettingsError = settings.SettingsError,
+    shell = require('./shell.js'),
     superagent = require('superagent'),
+    sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
-    updater = require('./updater.js'),
+    updateChecker = require('./updatechecker.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util'),
-    uuid = require('node-uuid'),
-    _ = require('underscore');
+    util = require('util');
 
-var SUDO = '/usr/bin/sudo',
-    TAR = os.platform() === 'darwin' ? '/usr/bin/tar' : '/bin/tar',
-    BACKUP_CMD = path.join(__dirname, 'scripts/backup.sh'),
-    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
+var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
+    REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
+    BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
+    BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
+    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
+
+var gAddMailDnsRecordsTimerId = null,
+    gCloudronDetails = null;            // cached cloudron details like region,size...
+
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+            callback();
+        });
+    };
+}
 
-var gBackupTimerId = null,
-    gAddMailDnsRecordsTimerId = null,
-    gGetCertificateTimerId = null,
-    gCachedIp = null;
 
 function CloudronError(reason, errorOrMessage) {
-    assert(typeof reason === 'string');
+    assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
 
     Error.call(this);
@@ -70,176 +95,197 @@ function CloudronError(reason, errorOrMessage) {
 util.inherits(CloudronError, Error);
 CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
+CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.APPSTORE_DOWN = 'Appstore Down';
+CloudronError.BAD_USERNAME = 'Bad username';
+CloudronError.BAD_EMAIL = 'Bad email';
+CloudronError.BAD_PASSWORD = 'Bad password';
+CloudronError.BAD_NAME = 'Bad name';
+CloudronError.BAD_STATE = 'Bad state';
+CloudronError.NOT_FOUND = 'Not found';
 
 function initialize(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    // every backup restarts the box. the setInterval is only needed should that fail for some reason
-    gBackupTimerId = setInterval(backup, 4 * 60 * 60 * 1000);
-
-    sendHeartBeat();
-
-    if (process.env.NODE_ENV !== 'test') {
+    if (process.env.BOX_ENV !== 'test') {
         addMailDnsRecords();
     }
 
+    // Send heartbeat once we are up and running, this speeds up the Cloudron creation, as otherwise we are bound to the cron.js settings
+    sendHeartbeat();
+
     callback(null);
 }
 
 function uninitialize(callback) {
-    assert(typeof callback === 'function');
-
-    clearInterval(gBackupTimerId);
-    gBackupTimerId = null;
+    assert.strictEqual(typeof callback, 'function');
 
     clearTimeout(gAddMailDnsRecordsTimerId);
     gAddMailDnsRecordsTimerId = null;
 
-    clearTimeout(gGetCertificateTimerId);
-    gGetCertificateTimerId = null;
-
-    gCachedIp = null;
-
     callback(null);
 }
 
-function activate(username, password, email, callback) {
-    assert(typeof username === 'string');
-    assert(typeof password === 'string');
-    assert(typeof email === 'string');
-    assert(typeof callback === 'function');
+function setTimeZone(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('setTimeZone ip:%s', ip);
+
+    superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
+        if (error || result.statusCode !== 200) {
+            debug('Failed to get geo location', error);
+            return callback(null);
+        }
+
+        if (!result.body.timezone) {
+            debug('No timezone in geoip response');
+            return callback(null);
+        }
+
+        debug('Setting timezone to ', result.body.timezone);
+
+        settings.setTimeZone(result.body.timezone, callback);
+    });
+}
+
+function activate(username, password, email, name, ip, callback) {
+    assert.strictEqual(typeof username, 'string');
+    assert.strictEqual(typeof password, 'string');
+    assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof ip, 'string');
+    assert(!name || typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('activating user:%s email:%s', username, email);
 
-    user.create(username, password, email, true /* admin */, function (error) {
-        if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error instanceof UserError) return callback(error);
+    setTimeZone(ip, function () { });
+
+    if (!name) name = settings.getDefaultSync(settings.CLOUDRON_NAME_KEY);
+
+    settings.setCloudronName(name, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_NAME));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        clientdb.getByAppId('webadmin', function (error, result) {
+        user.createOwner(username, password, email, function (error, userObject) {
+            if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
+            if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
+            if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
+            if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-            // Also generate a token so the admin creation can also act as a login
-            var token = tokendb.generateToken();
-            var expires = new Date(Date.now() + 60 * 60000).toUTCString(); // 1 hour
-
-            tokendb.add(token, username, result.id, expires, '*', function (error) {
+            clientdb.getByAppId('webadmin', function (error, result) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-                callback(null, { token: token, expires: expires });
+                // Also generate a token so the admin creation can also act as a login
+                var token = tokendb.generateToken();
+                var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+
+                tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+                    if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+                    callback(null, { token: token, expires: expires });
+                });
             });
         });
     });
 }
 
-function getBackupUrl(callback) {
-    assert(typeof callback === 'function');
-
-    if (!config.appServerUrl()) return callback(new Error('No appstore server url set'));
-    if (!config.token()) return callback(new Error('No appstore server token set'));
-
-    var url = config.appServerUrl() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
-
-    superagent.put(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
-        if (error) return callback(new Error('Error getting presigned backup url: ' + error.message));
-
-        if (result.statusCode !== 201 || !result.body || !result.body.url) return callback(new Error('Error getting presigned backup url : ' + result.statusCode));
-
-        return callback(null, result.body.url);
-    });
-}
-
-function backup(callback) {
-    assert(typeof callback === 'function');
-
-    getBackupUrl(function (error, url) {
-        if (error) return callback(new CloudronError(CloudronError.APPSTORE_DOWN, error.message));
-
-        debug('backup: url %s', url);
-
-        execFile(SUDO, [ BACKUP_CMD,  url ], { }, function (error) {
-            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-            return callback(null);
-        });
-    });
-}
-
-function getIp() {
-    if (gCachedIp) return gCachedIp;
-
-    var ifaces = os.networkInterfaces();
-    for (var dev in ifaces) {
-        if (dev.match(/^(en|eth|wlp).*/) === null) continue;
-
-        for (var i = 0; i < ifaces[dev].length; i++) {
-            if (ifaces[dev][i].family === 'IPv4') {
-                gCachedIp = ifaces[dev][i].address;
-                return gCachedIp;
-            }
-        }
-    }
-
-    return null;
-};
-
 function getStatus(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     userdb.count(function (error, count) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        callback(null, { activated: count !== 0, version: config.version() });
+        settings.getCloudronName(function (error, cloudronName) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                activated: count !== 0,
+                version: config.version(),
+                cloudronName: cloudronName
+            });
+        });
     });
 }
 
+function getCloudronDetails(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gCloudronDetails) return callback(null, gCloudronDetails);
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
+        .query({ token: config.token() })
+        .end(function (error, result) {
+            if (error) return callback(error);
+            if (result.status !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            gCloudronDetails = result.body.box;
+
+            return callback(null, gCloudronDetails);
+        });
+}
+
 function getConfig(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    callback(null, {
-        appServerUrl: config.appServerUrl(),
-        isDev: /dev/i.test(config.get('boxVersionsUrl')),
-        fqdn: config.fqdn(),
-        ip: getIp(),
-        version: config.version(),
-        update: updater.getUpdateInfo()
-    })
+    // TODO avoid pyramid of awesomeness with async
+    getCloudronDetails(function (error, result) {
+        if (error) {
+            console.error('Failed to fetch cloudron details.', error);
+
+            // set fallback values to avoid dependency on appstore
+            result = {
+                region: result ? result.region : null,
+                size: result ? result.size : null
+            };
+        }
+
+        settings.getCloudronName(function (error, cloudronName) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            settings.getDeveloperMode(function (error, developerMode) {
+                if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+                callback(null, {
+                    apiServerOrigin: config.apiServerOrigin(),
+                    webServerOrigin: config.webServerOrigin(),
+                    isDev: config.isDev(),
+                    fqdn: config.fqdn(),
+                    ip: sysinfo.getIp(),
+                    version: config.version(),
+                    update: updateChecker.getUpdateInfo(),
+                    progress: progress.get(),
+                    isCustomDomain: config.isCustomDomain(),
+                    developerMode: developerMode,
+                    region: result.region,
+                    size: result.size,
+                    cloudronName: cloudronName
+                });
+            });
+        });
+    });
 }
 
-function sendHeartBeat() {
-    var HEARTBEAT_INTERVAL = 1000 * 60;
+function sendHeartbeat() {
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
 
-    if (!config.appServerUrl()) {
-        debug('No appstore server url set. Not sending heartbeat.');
-        return;
-    }
-
-    if (!config.token()) {
-        debug('No appstore server token set. Not sending heartbeat.');
-        return;
-    }
-
-    var url = config.appServerUrl() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
-    debug('Sending heartbeat ' + url);
-
-    superagent.get(url).query({ token: config.token(), version: config.version() }).end(function (error, result) {
+    // TODO: this must be a POST
+    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
         if (error) debug('Error sending heartbeat.', error);
-        else if (result.statusCode !== 200) debug('Server responded to heartbeat with ' + result.statusCode);
-        else debug('Heartbeat successful');
-
-        setTimeout(sendHeartBeat, HEARTBEAT_INTERVAL);
+        else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
+        else debug('Heartbeat sent to %s', url);
     });
-};
+}
 
 function sendMailDnsRecordsRequest(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     var DKIM_SELECTOR = 'mail';
-    var DMARC_REPORT_EMAIL = 'girish@forwardbias.in';
+    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
 
-    var dkimPublicKeyFile = path.join(paths.HARAKA_CONFIG_DIR, 'dkim/' + config.fqdn() + '/public');
+    var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
 
     if (publicKey === null) return callback(new Error('Error reading dkim public key'));
@@ -250,7 +296,7 @@ function sendMailDnsRecordsRequest(callback) {
     // note that dmarc requires special DNS records for external RUF and RUA
     var records = [
         // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
-        { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + getIp() + ' ~all"' },
+        { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
         // t=s limits the domainkey to this domain and not it's subdomains
         { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', value: '"v=DKIM1; t=s; p=' + publicKey + '"' },
         // DMARC requires special setup if report email id is in different domain
@@ -260,7 +306,7 @@ function sendMailDnsRecordsRequest(callback) {
     debug('sendMailDnsRecords request:%s', JSON.stringify(records));
 
     superagent
-        .post(config.appServerUrl() + '/api/v1/subdomains')
+        .post(config.apiServerOrigin() + '/api/v1/subdomains')
         .set('Accept', 'application/json')
         .query({ token: config.token() })
         .send({ records: records })
@@ -278,9 +324,6 @@ function sendMailDnsRecordsRequest(callback) {
 }
 
 function addMailDnsRecords() {
-    // TODO assert replaced with a non fatal return, for local development
-    if (!config.token()) return;
-
     if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
 
     sendMailDnsRecordsRequest(function (error, ids) {
@@ -296,8 +339,9 @@ function addMailDnsRecords() {
 }
 
 function setCertificate(certificate, key, callback) {
-    assert(typeof certificate === 'string');
-    assert(typeof key === 'string');
+    assert.strictEqual(typeof certificate, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('Updating certificates');
 
@@ -309,10 +353,283 @@ function setCertificate(certificate, key, callback) {
         return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
     }
 
-    execFile(SUDO, [ RELOAD_NGINX_CMD ], { timeout: 10000 }, function (error) {
+    shell.sudo('setCertificate', [ RELOAD_NGINX_CMD ], function (error) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         return callback(null);
     });
 }
 
+function reboot(callback) {
+    shell.sudo('reboot', [ REBOOT_CMD ], callback);
+}
+
+function migrate(size, region, callback) {
+    assert.strictEqual(typeof size, 'string');
+    assert.strictEqual(typeof region, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_MIGRATE);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    function unlock(error) {
+        if (error) {
+            debug('Failed to migrate', error);
+            locker.unlock(locker.OP_MIGRATE);
+        } else {
+            debug('Migration initiated successfully');
+            // do not unlock; cloudron is migrating
+        }
+
+        return;
+    }
+
+    // initiate the migration in the background
+    backupBoxAndApps(function (error, restoreKey) {
+        if (error) return unlock(error);
+
+        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
+
+        superagent
+          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+          .query({ token: config.token() })
+          .send({ size: size, region: region, restoreKey: restoreKey })
+          .end(function (error, result) {
+            if (error) return unlock(error);
+            if (result.status === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+            if (result.status === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+            if (result.status !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+
+            return unlock(null);
+        });
+    });
+
+    callback(null);
+}
+
+function update(boxUpdateInfo, callback) {
+    assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!boxUpdateInfo) return callback(null);
+
+    var error = locker.lock(locker.OP_BOX_UPDATE);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    // initiate the update/upgrade but do not wait for it
+    if (boxUpdateInfo.upgrade) {
+        debug('Starting upgrade');
+        doUpgrade(boxUpdateInfo, function (error) {
+            if (error) {
+                debug('Upgrade failed with error: %s', error);
+                locker.unlock(locker.OP_BOX_UPDATE);
+            }
+        });
+    } else {
+        debug('Starting update');
+        doUpdate(boxUpdateInfo, function (error) {
+            if (error) {
+                debug('Update failed with error: %s', error);
+                locker.unlock(locker.OP_BOX_UPDATE);
+            }
+        });
+    }
+
+    callback(null);
+}
+
+function doUpgrade(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
+
+    function upgradeError(e) {
+        progress.set(progress.UPDATE, -1, e.message);
+        callback(e);
+    }
+
+    progress.set(progress.UPDATE, 5, 'Create app and box backup for upgrade');
+
+    backupBoxAndApps(function (error) {
+        if (error) return upgradeError(error);
+
+        superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
+          .query({ token: config.token() })
+          .send({ version: boxUpdateInfo.version })
+          .end(function (error, result) {
+            if (error) return upgradeError(new Error('Error making upgrade request: ' + error));
+            if (result.status !== 202) return upgradeError(new Error('Server not ready to upgrade: ' + result.body));
+
+            progress.set(progress.UPDATE, 10, 'Updating base system');
+
+            // no need to unlock since this is the last thing we ever do on this box
+
+            callback(null);
+        });
+    });
+}
+
+function doUpdate(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo && typeof boxUpdateInfo === 'object');
+
+    function updateError(e) {
+        progress.set(progress.UPDATE, -1, e.message);
+        callback(e);
+    }
+
+    progress.set(progress.UPDATE, 5, 'Create box backup for update');
+
+    backupBox(function (error) {
+        if (error) return updateError(error);
+
+        // fetch a signed sourceTarballUrl
+        superagent.get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/sourcetarballurl')
+          .query({ token: config.token(), boxVersion: boxUpdateInfo.version })
+          .end(function (error, result) {
+            if (error) return updateError(new Error('Error fetching sourceTarballUrl: ' + error));
+            if (result.status !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.status));
+            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + result.body));
+
+            // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
+            var args = {
+                sourceTarballUrl: result.body.url,
+
+                // IMPORTANT: if you change this, fix up argparser.sh as well. keep these sorted for readability
+                data: {
+                    apiServerOrigin: config.apiServerOrigin(),
+                    boxVersionsUrl: config.get('boxVersionsUrl'),
+                    fqdn: config.fqdn(),
+                    isCustomDomain: config.isCustomDomain(),
+                    restoreKey: null,
+                    restoreUrl: null,
+                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                    token: config.token(),
+                    version: boxUpdateInfo.version,
+                    webServerOrigin: config.webServerOrigin()
+                }
+            };
+
+            debug('updating box %j', args);
+
+            superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
+                if (error) return updateError(error);
+                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + result.body));
+
+                progress.set(progress.UPDATE, 10, 'Updating cloudron software');
+
+                callback(null);
+            });
+        });
+
+        // Do not add any code here. The installer script will stop the box code any instant
+    });
+}
+
+function backup(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = locker.lock(locker.OP_FULL_BACKUP);
+    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
+
+    // start the backup operation in the background
+    backupBoxAndApps(function (error) {
+        if (error) console.error('backup failed.', error);
+
+        locker.unlock(locker.OP_FULL_BACKUP);
+    });
+
+    callback(null);
+}
+
+function ensureBackup(callback) {
+    callback = callback || function () { };
+
+    backups.getAllPaged(1, 1, function (error, backups) {
+        if (error) {
+            debug('Unable to list backups', error);
+            return callback(error); // no point trying to backup if appstore is down
+        }
+
+        if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
+            debug('Previous backup was %j, no need to backup now', backups[0]);
+            return callback(null);
+        }
+
+        backup(callback);
+    });
+}
+
+function backupBoxWithAppBackupIds(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+
+    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        debug('backup: url %s', result.url);
+
+        async.series([
+            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
+        ], function (error) {
+            if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+            debug('backup: successful');
+
+            callback(null, result.id);
+        });
+    });
+}
+
+// this function expects you to have a lock
+function backupBox(callback) {
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
+        appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+
+        backupBoxWithAppBackupIds(appBackupIds, callback);
+    });
+}
+
+// this function expects you to have a lock
+function backupBoxAndApps(callback) {
+    callback = callback || function () { }; // callback can be empty for timer triggered backup
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
+        var processed = 0;
+        var step = 100/(allApps.length+1);
+
+        progress.set(progress.BACKUP, processed, '');
+
+        async.mapSeries(allApps, function iterator(app, iteratorCallback) {
+            ++processed;
+
+            apps.backupApp(app, app.manifest.addons, function (error, backupId) {
+                progress.set(progress.BACKUP, step * processed, 'Backing up app at ' + app.location);
+
+                if (error && error.reason === AppsError.BAD_STATE) {
+                    debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);
+                    backupId = app.lastBackupId;
+                }
+
+                return iteratorCallback(null, backupId);
+            });
+        }, function appsBackedUp(error, backupIds) {
+            if (error) {
+                progress.set(progress.BACKUP, 100, error.message);
+                return callback(error);
+            }
+
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+
+            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
+                progress.set(progress.BACKUP, 100, error ? error.message : '');
+                callback(error, restoreKey);
+            });
+        });
+    });
+}

src/collectd.config.ejs

@@ -30,13 +30,3 @@ LoadPlugin "table"
     </Result>
   </Table>
 </Plugin>
-
-LoadPlugin "filecount"
-<Plugin "filecount">
-  <Directory "/home/yellowtent/data/appdata/<%= appId %>">
-    Instance "<%= appId %>-appdata"
-    IncludeHidden true
-    Recursive true
-  </Directory>
-</Plugin>
-

src/config.js

@@ -2,45 +2,52 @@
 
 'use strict';
 
-var path = require('path'),
-    fs = require('fs'),
-    safe = require('safetydance'),
-    assert = require('assert'),
-    _ = require('underscore'),
-    path = require('path'),
-    mkdirp = require('mkdirp');
-
 exports = module.exports = {
     baseDir: baseDir,
+
+    // values set here will be lost after a upgrade/update. use the sqlite database
+    // for persistent values that need to be backed up
     get: get,
     set: set,
 
     // ifdefs to check environment
-    CLOUDRON: process.env.NODE_ENV === 'cloudron',
-    TEST: process.env.NODE_ENV === 'test',
-    LOCAL: process.env.NODE_ENV === 'local' || !process.env.NODE_ENV,
+    CLOUDRON: process.env.BOX_ENV === 'cloudron',
+    TEST: process.env.BOX_ENV === 'test',
 
     // convenience getters
-    appServerUrl: appServerUrl,
+    apiServerOrigin: apiServerOrigin,
+    webServerOrigin: webServerOrigin,
     fqdn: fqdn,
     token: token,
     version: version,
     isCustomDomain: isCustomDomain,
+    database: database,
 
     // these values are derived
     adminOrigin: adminOrigin,
     appFqdn: appFqdn,
-    zoneName: zoneName
+    zoneName: zoneName,
+
+    isDev: isDev,
+
+    // for testing resets to defaults
+    _reset: initConfig
 };
 
+var assert = require('assert'),
+    constants = require('./constants.js'),
+    fs = require('fs'),
+    path = require('path'),
+    safe = require('safetydance'),
+    _ = require('underscore');
+
 var homeDir = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
 
 var data = { };
 
 function baseDir() {
     if (exports.CLOUDRON) return homeDir;
-    if (exports.TEST) return path.join(homeDir, '.yellowtenttest');
-    if (exports.LOCAL) return path.join(homeDir, '.yellowtent');
+    if (exports.TEST) return path.join(homeDir, '.cloudron_test');
 }
 
 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
@@ -49,30 +56,39 @@ function saveSync() {
     fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
 
-(function initConfig() {
+function initConfig() {
     // setup defaults
-    if (exports.CLOUDRON) {
-        data.port = 3000;
-        data.appServerUrl = process.env.APP_SERVER_URL || null; // APP_SERVER_URL is set during bootstrap in the box's supervisor manifest
-    } else if (exports.TEST) {
-        data.port = 5454;
-        data.appServerUrl = 'http://localhost:6060'; // hock doesn't support https
-    } else if (exports.LOCAL) {
-        data.port = 3000;
-        data.appServerUrl = 'http://localhost:5050';
-    } else {
-        assert(false, 'Unknown environment. This should not happen!');
-    }
-
     data.fqdn = 'localhost';
 
     data.token = null;
     data.mailServer = null;
-    data.mailUsername = null;
+    data.adminEmail = null;
     data.mailDnsRecordIds = [ ];
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
+    data.webServerOrigin = null;
+    data.internalPort = 3001;
+    data.ldapPort = 3002;
+
+    if (exports.CLOUDRON) {
+        data.port = 3000;
+        data.apiServerOrigin = null;
+        data.database = null;
+    } else if (exports.TEST) {
+        data.port = 5454;
+        data.apiServerOrigin = 'http://localhost:6060'; // hock doesn't support https
+        data.database = {
+            hostname: 'localhost',
+            username: 'root',
+            password: '',
+            port: 3306,
+            name: 'boxtest'
+        };
+        data.token = 'APPSTORE_TOKEN';
+    } else {
+        assert(false, 'Unknown environment. This should not happen!');
+    }
 
     if (safe.fs.existsSync(cloudronConfigFileName)) {
         var existingData = safe.JSON.parse(safe.fs.readFileSync(cloudronConfigFileName, 'utf8'));
@@ -80,9 +96,10 @@ function saveSync() {
         return;
     }
 
-    mkdirp.sync(path.dirname(cloudronConfigFileName));
     saveSync();
-})();
+}
+
+initConfig();
 
 // set(obj) or set(key, value)
 function set(key, value) {
@@ -93,33 +110,39 @@ function set(key, value) {
             data[k] = obj[k];
         }
     } else {
-        assert(key in data, 'config.js is missing key "' + key + '"');
-        data[key] = value;
+        data = safe.set(data, key, value);
     }
     saveSync();
 }
 
 function get(key) {
-    assert(typeof key === 'string');
+    assert.strictEqual(typeof key, 'string');
 
     return safe.query(data, key);
 }
 
-function appServerUrl() {
-    return get('appServerUrl');
+function apiServerOrigin() {
+    return get('apiServerOrigin');
+}
+
+function webServerOrigin() {
+    return get('webServerOrigin');
 }
 
 function fqdn() {
     return get('fqdn');
 }
 
+// keep this in sync with start.sh admin.conf generation code
 function appFqdn(location) {
-    assert(typeof location === 'string');
+    assert.strictEqual(typeof location, 'string');
+
+    if (location === '') return fqdn();
     return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
 }
 
 function adminOrigin() {
-    return 'https://' + appFqdn('admin');
+    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
 
 function token() {
@@ -141,3 +164,11 @@ function zoneName() {
     return fqdn().substr(fqdn().indexOf('.') + 1);
 }
 
+function database() {
+    return get('database');
+}
+
+function isDev() {
+    return /dev/i.test(get('boxVersionsUrl'));
+}
+

src/constants.js

@@ -0,0 +1,16 @@
+'use strict';
+
+ // default admin installation location. keep in sync with ADMIN_LOCATION in setup/start.sh and BOX_ADMIN_LOCATION in appstore constants.js
+exports = module.exports = {
+    ADMIN_LOCATION: 'my',
+    API_LOCATION: 'api', // this is unused but reserved for future use (#403)
+    ADMIN_NAME: 'Settings',
+
+    ADMIN_CLIENT_ID: 'webadmin', // oauth client id
+    ADMIN_APPID: 'admin', // admin appid (settingsdb)
+
+    TEST_NAME: 'Test',
+    TEST_LOCATION: '',
+    TEST_CLIENT_ID: 'test'
+};
+

src/cron.js

@@ -0,0 +1,141 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+    uninitialize: uninitialize
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    cloudron = require('./cloudron.js'),
+    CronJob = require('cron').CronJob,
+    debug = require('debug')('box:cron'),
+    settings = require('./settings.js'),
+    updateChecker = require('./updatechecker.js');
+
+var gAutoupdaterJob = null,
+    gBoxUpdateCheckerJob = null,
+    gAppUpdateCheckerJob = null,
+    gHeartbeatJob = null,
+    gBackupJob = null;
+
+var gInitialized = false;
+
+var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+
+// cron format
+// Seconds: 0-59
+// Minutes: 0-59
+// Hours: 0-23
+// Day of Month: 1-31
+// Months: 0-11
+// Day of Week: 0-6
+
+function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gInitialized) return callback();
+
+    settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
+    settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
+
+    gInitialized = true;
+
+    recreateJobs(callback);
+}
+
+function recreateJobs(unusedTimeZone, callback) {
+    if (typeof unusedTimeZone === 'function') callback = unusedTimeZone;
+
+    settings.getAll(function (error, allSettings) {
+        if (gHeartbeatJob) gHeartbeatJob.stop();
+        gHeartbeatJob = new CronJob({
+            cronTime: '00 */1 * * * *', // every minute
+            onTick: cloudron.sendHeartbeat,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gBackupJob) gBackupJob.stop();
+        gBackupJob = new CronJob({
+            cronTime: '00 00 */4 * * *', // every 4 hours
+            onTick: cloudron.ensureBackup,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
+        gBoxUpdateCheckerJob = new CronJob({
+            cronTime: '00 */10 * * * *', // every 10 minutes
+            onTick: updateChecker.checkBoxUpdates,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
+        gAppUpdateCheckerJob = new CronJob({
+            cronTime: '00 */10 * * * *', // every 10 minutes
+            onTick: updateChecker.checkAppUpdates,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY]);
+
+        if (callback) callback();
+    });
+}
+
+function autoupdatePatternChanged(pattern) {
+    assert.strictEqual(typeof pattern, 'string');
+
+    debug('Auto update pattern changed to %s', pattern);
+
+    if (gAutoupdaterJob) gAutoupdaterJob.stop();
+
+    if (pattern === 'never') return;
+
+    gAutoupdaterJob = new CronJob({
+        cronTime: pattern,
+        onTick: function() {
+            var updateInfo = updateChecker.getUpdateInfo();
+            if (updateInfo.box) {
+                debug('Starting autoupdate to %j', updateInfo.box);
+                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+            } else if (updateInfo.apps) {
+                debug('Starting app update to %j', updateInfo.apps);
+                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+            } else {
+                debug('No auto updates available');
+            }
+        },
+        start: true,
+        timeZone: gBoxUpdateCheckerJob.cronTime.timeZone // hack
+    });
+}
+
+function uninitialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!gInitialized) return callback();
+
+    if (gAutoupdaterJob) gAutoupdaterJob.stop();
+    gAutoupdaterJob = null;
+
+    gBoxUpdateCheckerJob.stop();
+    gBoxUpdateCheckerJob = null;
+
+    gAppUpdateCheckerJob.stop();
+    gAppUpdateCheckerJob = null;
+
+    gHeartbeatJob.stop();
+    gHeartbeatJob = null;
+
+    gBackupJob.stop();
+    gBackupJob = null;
+
+    gInitialized = false;
+
+    callback();
+}
+

src/database.js

@@ -1,123 +1,202 @@
-/* jslint node:true */
+/* jslint node: true */
 
 'use strict';
 
-var assert = require('assert'),
-    async = require('async'),
-    config = require('../config.js'),
-    DatabaseError = require('./databaseerror.js'),
-    debug = require('debug')('box:database'),
-    paths = require('./paths.js'),
-    sqlite3 = require('sqlite3');
-
 exports = module.exports = {
     initialize: initialize,
     uninitialize: uninitialize,
-    removePrivates: removePrivates,
+    query: query,
+    transaction: transaction,
+
     beginTransaction: beginTransaction,
     rollback: rollback,
     commit: commit,
-    clear: clear,
 
-    get: get,
-    all: all,
-    run: run
+    _clear: clear
 };
 
-var gConnectionPool = [ ], // used to track active transactions
-    gDatabase = null;
+var assert = require('assert'),
+    async = require('async'),
+    once = require('once'),
+    config = require('./config.js'),
+    mysql = require('mysql'),
+    util = require('util');
 
-var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var gConnectionPool = null,
+    gDefaultConnection = null;
 
-function initialize(callback) {
-    gDatabase = new sqlite3.Database(paths.DATABASE_FILENAME);
-    gDatabase.on('error', function (error) {
-        console.error('Database error in ' + paths.DATABASE_FILENAME + ':', error);
+function initialize(options, callback) {
+    if (typeof options === 'function') {
+        callback = options;
+        options = {
+            connectionLimit: 5
+        };
+    }
+
+    assert.strictEqual(typeof options.connectionLimit, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gConnectionPool !== null) return callback(null);
+
+    gConnectionPool  = mysql.createPool({
+        connectionLimit: options.connectionLimit,
+        host: config.database().hostname,
+        user: config.database().username,
+        password: config.database().password,
+        port: config.database().port,
+        database: config.database().name,
+        multipleStatements: false,
+        ssl: false
     });
 
-    gDatabase.run('PRAGMA busy_timeout=5000', callback);
+    reconnect(callback);
 }
 
 function uninitialize(callback) {
-    assert(typeof callback === 'function');
+    if (gConnectionPool) {
+        gConnectionPool.end(callback);
+        gConnectionPool = null;
+    } else {
+        callback(null);
+    }
+}
 
-    debug('Closing database');
-    gDatabase.close();
-    gDatabase = null;
+function setupConnection(connection, callback) {
+    assert.strictEqual(typeof connection, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    debug('Closing %d active transactions', gConnectionPool.length);
-    gConnectionPool.forEach(function (conn) { conn.close(); });
-    gConnectionPool = [ ];
+    connection.on('error', console.error);
 
-    callback(null);
+    async.series([
+        connection.query.bind(connection, 'USE ' + config.database().name),
+        connection.query.bind(connection, 'SET SESSION sql_mode = \'strict_all_tables\'')
+    ], function (error) {
+        connection.removeListener('error', console.error);
+
+        if (error) connection.release();
+
+        callback(error);
+    });
+}
+
+function reconnect(callback) {
+    callback = callback ? once(callback) : function () {};
+
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) {
+            console.error('Unable to reestablish connection to database. Try again in a bit.', error.message);
+            return setTimeout(reconnect.bind(null, callback), 1000);
+        }
+
+        connection.on('error', function (error) {
+            // by design, we catch all normal errors by providing callbacks.
+            // this function should be invoked only when we have no callbacks pending and we have a fatal error
+            assert(error.fatal, 'Non-fatal error on connection object');
+
+            console.error('Unhandled mysql connection error.', error);
+
+            // This is most likely an issue an can cause double callbacks from reconnect()
+            setTimeout(reconnect.bind(null, callback), 1000);
+        });
+
+        setupConnection(connection, function (error) {
+            if (error) return setTimeout(reconnect.bind(null, callback), 1000);
+
+            gDefaultConnection = connection;
+
+            callback(null);
+        });
+    });
 }
 
 function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    // the clear funcs don't completely clear the db, they leave the migration code defaults
     async.series([
-        require('./appdb.js').clear,
-        require('./authcodedb.js').clear,
-        require('./clientdb.js').clear,
-        require('./tokendb.js').clear,
-        require('./userdb.js').clear
+        require('./appdb.js')._clear,
+        require('./authcodedb.js')._clear,
+        require('./clientdb.js')._clear,
+        require('./tokendb.js')._clear,
+        require('./userdb.js')._clear,
+        require('./settingsdb.js')._clear
     ], callback);
 }
 
-function beginTransaction() {
-    var conn = new sqlite3.Database(paths.DATABASE_FILENAME);
-    conn._started = Date.now();
-    conn._slowWarningIntervalId = setInterval((function () {
-        debug('Transaction running for %d msecs', Date.now() - this._started);
-    }).bind(conn), 2000);
+function beginTransaction(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    gConnectionPool.push(conn);
-    conn.serialize();
-    conn.run('PRAGMA busy_timeout=5000', NOOP_CALLBACK);
-    conn.run('BEGIN TRANSACTION', NOOP_CALLBACK);
-    return conn;
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) return callback(error);
+
+        setupConnection(connection, function (error) {
+            if (error) return callback(error);
+
+            connection.beginTransaction(function (error) {
+                if (error) return callback(error);
+
+                return callback(null, connection);
+            });
+        });
+    });
 }
 
-function rollback(conn, callback) {
-    gConnectionPool.splice(gConnectionPool.indexOf(conn), 1);
-    conn.run('ROLLBACK', NOOP_CALLBACK);
-    clearInterval(conn._slowWarningIntervalId);
-    debug('Transaction took %d msecs', Date.now() - conn._started);
-    conn.close(); // close waits for pending statements
-    if (callback) callback();
-}
+function rollback(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-function commit(conn, callback) {
-    gConnectionPool.splice(gConnectionPool.indexOf(conn), 1);
-    conn.run('COMMIT', function (error) {
-        clearInterval(conn._slowWarningIntervalId);
-        debug('Transaction took %d msecs', Date.now() - conn._started);
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    connection.rollback(function (error) {
+        if (error) console.error(error); // can this happen?
 
+        connection.release();
         callback(null);
     });
-    conn.close(); // close waits for pending statements
 }
 
-function removePrivates(obj) {
-    var res = { };
+// FIXME: if commit fails, is it supposed to return an error ?
+function commit(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
 
-    for (var p in obj) {
-        if (!obj.hasOwnProperty(p)) continue;
-        if (p.substring(0, 1) === '_') continue;
-        res[p] = obj[p]; // ## make deep copy?
-    }
+    connection.commit(function (error) {
+        if (error) return rollback(connection, callback);
 
-    return res;
+        connection.release();
+        return callback(null);
+    });
 }
 
-function get() {
-    return gDatabase.get.apply(gDatabase, arguments);
+function query() {
+    var args = Array.prototype.slice.call(arguments);
+    var callback = args[args.length - 1];
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gDefaultConnection === null) return callback(new Error('No connection to database'));
+
+    args[args.length -1 ] = function (error, result) {
+        if (error && error.fatal) {
+            gDefaultConnection = null;
+            setTimeout(reconnect, 1000);
+        }
+
+        callback(error, result);
+    };
+
+    gDefaultConnection.query.apply(gDefaultConnection, args);
 }
 
-function all() {
-    return gDatabase.all.apply(gDatabase, arguments);
-}
-
-function run() {
-    return gDatabase.run.apply(gDatabase, arguments);
+function transaction(queries, callback) {
+    assert(util.isArray(queries));
+    assert.strictEqual(typeof callback, 'function');
+
+    beginTransaction(function (error, conn) {
+        if (error) return callback(error);
+
+        async.mapSeries(queries, function iterator(query, done) {
+            conn.query(query.query, query.args, done);
+        }, function seriesDone(error, results) {
+            if (error) return rollback(conn, callback.bind(null, error));
+
+            commit(conn, callback.bind(null, null, results));
+        });
+    });
 }
 

src/databaseerror.js

@@ -2,20 +2,20 @@
 
 'use strict';
 
+exports = module.exports = DatabaseError;
+
 var assert = require('assert'),
     util = require('util');
 
-module.exports = exports = DatabaseError;
-
 function DatabaseError(reason, errorOrMessage) {
-    assert(typeof reason === 'string');
-    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined' || errorOrMessage === null);
 
     Error.call(this);
     Error.captureStackTrace(this, this.constructor);
 
     this.reason = reason;
-    if (typeof errorOrMessage === 'undefined') {
+    if (typeof errorOrMessage === 'undefined' || errorOrMessage === null) {
         this.message = reason;
     } else if (typeof errorOrMessage === 'string') {
         this.message = errorOrMessage;
@@ -29,5 +29,4 @@ util.inherits(DatabaseError, Error);
 DatabaseError.INTERNAL_ERROR = 'Internal error';
 DatabaseError.ALREADY_EXISTS = 'Entry already exist';
 DatabaseError.NOT_FOUND = 'Record not found';
-DatabaseError.RECORD_SCHEMA = 'Record does not match the schema';
-DatabaseError.FIELD_ERROR = 'Invalid field';
+DatabaseError.BAD_FIELD = 'Invalid field';

src/developer.js

@@ -0,0 +1,85 @@
+/* jslint node: true */
+
+'use strict';
+
+exports = module.exports = {
+    DeveloperError: DeveloperError,
+
+    enabled: enabled,
+    setEnabled: setEnabled,
+    issueDeveloperToken: issueDeveloperToken,
+    getNonApprovedApps: getNonApprovedApps
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    tokendb = require('./tokendb.js'),
+    settings = require('./settings.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+function DeveloperError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(DeveloperError, Error);
+DeveloperError.INTERNAL_ERROR = 'Internal Error';
+
+function enabled(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.getDeveloperMode(function (error, enabled) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        callback(null, enabled);
+    });
+}
+
+function setEnabled(enabled, callback) {
+    assert.strictEqual(typeof enabled, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
+
+    settings.setDeveloperMode(enabled, function (error) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function issueDeveloperToken(user, callback) {
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var token = tokendb.generateToken();
+    var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+
+    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'apps,settings,roleDeveloper', function (error) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+
+        callback(null, { token: token, expiresAt: expiresAt });
+    });
+}
+
+function getNonApprovedApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
+    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
+        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        if (result.status !== 200) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+
+        callback(null, result.body.apps || []);
+    });
+}

src/digitalocean.js

@@ -2,29 +2,23 @@
 
 'use strict';
 
-var assert = require('assert'),
-    debug = require('debug')('box:digitalocean'),
-    config = require('../config.js'),
-    dns = require('native-dns');
-
 exports = module.exports = {
     checkPtrRecord: checkPtrRecord
 };
 
+var assert = require('assert'),
+    debug = require('debug')('box:digitalocean'),
+    dns = require('native-dns');
+
 function checkPtrRecord(ip, fqdn, callback) {
     assert(ip === null || typeof ip === 'string');
-    assert(typeof fqdn === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof fqdn, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
     debug('checkPtrRecord: ' + ip);
 
     if (!ip) return callback(new Error('Network down'));
 
-    if (config.LOCAL) {
-        debug('checkPtrRecord disabled in local mode.');
-        return callback(null, true);
-    }
-
     dns.resolve4('ns1.digitalocean.com', function (error, rdnsIps) {
         if (error || rdnsIps.length === 0) return callback(new Error('Failed to query DO DNS'));
 

src/docker.js

@@ -1,7 +1,6 @@
 'use strict';
 
-var assert = require('assert'),
-    Docker = require('dockerode'),
+var Docker = require('dockerode'),
     fs = require('fs'),
     os = require('os'),
     path = require('path'),
@@ -11,7 +10,7 @@ exports = module.exports = (function () {
     var docker;
     var options = connectOptions(); // the real docker
 
-    if (process.env.NODE_ENV === 'test') {
+    if (process.env.BOX_ENV === 'test') {
         // test code runs a docker proxy on this port
         docker = new Docker({ host: 'http://localhost', port: 5687 });
     } else {

src/ldap.js

@@ -0,0 +1,108 @@
+'use strict';
+
+exports = module.exports = {
+    start: start
+};
+
+var assert = require('assert'),
+    config = require('./config.js'),
+    debug = require('debug')('box:ldap'),
+    user = require('./user.js'),
+    UserError = user.UserError,
+    ldap = require('ldapjs');
+
+var gServer = null;
+
+var NOOP = function () {};
+
+var gLogger = {
+    trace: NOOP,
+    debug: NOOP,
+    info: debug,
+    warn: debug,
+    error: console.error,
+    fatal: console.error
+};
+
+function start(callback) {
+    assert(typeof callback === 'function');
+
+    gServer = ldap.createServer({ log: gLogger });
+
+    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
+        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+
+        user.list(function (error, result){
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // send user objects
+            result.forEach(function (entry) {
+                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+                var tmp = {
+                    dn: dn.toString(),
+                    attributes: {
+                        objectclass: ['user'],
+                        objectcategory: 'person',
+                        cn: entry.id,
+                        uid: entry.id,
+                        mail: entry.email,
+                        displayname: entry.username,
+                        username: entry.username,
+                        samaccountname: entry.username      // to support ActiveDirectory clients
+                    }
+                };
+
+                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
+                    res.send(tmp);
+                    debug('ldap user send:', tmp);
+                }
+            });
+
+            res.end();
+        });
+    });
+
+    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
+        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+
+        user.list(function (error, result){
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we only have an admin group
+            var dn = ldap.parseDN('cn=admin,ou=groups,dc=cloudron');
+
+            var tmp = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: 'admin',
+                    memberuid: result.filter(function (entry) { return entry.admin; }).map(function(entry) { return entry.id; })
+                }
+            };
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
+                res.send(tmp);
+                debug('ldap group send:', tmp);
+            }
+
+            res.end();
+        });
+    });
+
+    gServer.bind('dc=cloudron', function(req, res, next) {
+        debug('ldap bind: %s', req.dn.toString());
+
+        if (!req.dn.rdns[0].cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+        user.verify(req.dn.rdns[0].cn, req.credentials || '', function (error, result) {
+            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+            if (error) return next(new ldap.OperationsError(error));
+
+            res.end();
+        });
+    });
+
+    gServer.listen(config.get('ldapPort'), callback);
+}

src/locker.js

@@ -0,0 +1,55 @@
+'use strict';
+
+var assert = require('assert'),
+    debug = require('debug')('box:locker'),
+    EventEmitter = require('events').EventEmitter,
+    util = require('util');
+
+function Locker() {
+    this._operation = null;
+    this._timestamp = null;
+    this._watcherId = -1;
+}
+util.inherits(Locker, EventEmitter);
+
+// these are mutually exclusive operations
+Locker.prototype.OP_BOX_UPDATE = 'box_update';
+Locker.prototype.OP_FULL_BACKUP = 'full_backup';
+Locker.prototype.OP_APPTASK = 'apptask';
+Locker.prototype.OP_MIGRATE = 'migrate';
+
+Locker.prototype.lock = function (operation) {
+    assert.strictEqual(typeof operation, 'string');
+
+    if (this._operation !== null) return new Error('Already locked for ' + this._operation);
+
+    this._operation = operation;
+    this._timestamp = new Date();
+    var that = this;
+    this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
+
+    debug('Acquired : %s', this._operation);
+
+    this.emit('locked', this._operation);
+
+    return null;
+};
+
+Locker.prototype.unlock = function (operation) {
+    assert.strictEqual(typeof operation, 'string');
+
+    if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
+
+    debug('Released : %s', this._operation);
+
+    this._operation = null;
+    this._timestamp = null;
+    clearInterval(this._watcherId);
+    this._watcherId = -1;
+
+    this.emit('unlocked', operation);
+
+    return null;
+};
+
+exports = module.exports = new Locker();

src/mail_templates/app_down.ejs

@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+The application titled '<%= title %>' that you installed at <%= appFqdn %>
+is not responding.
+
+This is most likely a problem in the application. Please report this issue to
+support@cloudron.io (by forwarding this email).
+
+You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
+
+Thank you,
+Application WatchDog
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/app_down_html.ejs

@@ -1 +0,0 @@
-Application <%= location %> is unreachable, just letting you know.

src/mail_templates/app_down_text.ejs

@@ -1 +0,0 @@
-Application <%= location %> is unreachable, just letting you know.

src/mail_templates/app_update_available.ejs

@@ -0,0 +1,15 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+A new version of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+
+Please update at your convenience at <%= webadminUrl %>.
+
+Thank you,
+Update Manager
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/box_update_available.ejs

@@ -0,0 +1,20 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+A new version of Cloudron <%= fqdn %> is available!
+
+Please update at your convenience at <%= webadminUrl %>.
+
+Changelog:
+<% for (var i = 0; i < changelog.length; i++) { %>
+  * <%- changelog[i] %>
+<% } %>
+
+Thank you,
+Update Manager
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/crash_notification.ejs

@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+
+unfortunately the <%= program %> on <%= fqdn %> crashed unexpectedly!
+
+Please see some excerpt of the logs below.
+
+Thank you,
+Your Cloudron
+
+-------------------------------------
+
+<%= context %>
+
+<% } else { %>
+
+<% } %>
+