Version 0.6.2 changes

The main issue is that multiple cloudron exec sessions do not
share the same rootfs. Which makes it annoying to debug.

We also have some nginx timeout which drops you out of exec
now and then resulting in loss of all state.

.gitattributes

@@ -1,10 +1,7 @@
-# Skip files when using git archive
+# following files are skipped when exporting using git archive
+/release export-ignore
+/admin export-ignore
+test export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
-/release export-ignore
-/scripts export-ignore
-test export-ignore
-/webadmin/src export-ignore
-/webadmin/deploymentConfig.json export-ignore
-/gulpfile.json export-ignore
 

.gitignore

@@ -1,20 +1,11 @@
 node_modules/
 coverage/
 docs/
+webadmin/dist/
+setup/splash/website/
+installer/src/certs/server.key
 
-# vim swam files
+# vim swap files
 *.swp
 
-# supervisor
-supervisord.pid
-supervisord.log
-
-# nginx
-nginx/*.log
-nginx/*.pid
-nginx/naked_domain.conf
-nginx/applications/
-
-# release files
-release/versions-dev.json
 

CHANGES

@@ -0,0 +1,374 @@
+[0.0.1]
+- Hot Chocolate
+
+[0.0.2]
+- Hotfix appstore ui in webadim
+
+[0.0.3]
+- Tall Pike
+
+[0.0.4]
+- This will be 0.0.4 changes
+
+[0.0.5]
+- App install/configure route fixes
+
+[0.0.6]
+- Not sure what happenned here
+
+[0.0.7]
+- resetToken is now sent as part of create user
+- Same as 0.0.7 which got released by mistake
+
+[0.0.8]
+- Manifest changes
+
+[0.0.9]
+- Fix app restore
+- Fix backup issues
+
+[0.0.10]
+- Unknown orchestra
+
+[0.0.11]
+- Add ldap addon
+
+[0.0.12]
+- Support OAuth2 state
+
+[0.0.13]
+- Use docker image from cloudron repository
+
+[0.0.14]
+- Improve setup flow
+
+[0.0.15]
+- Improved Appstore view
+
+[0.0.16]
+- Improved Backup approach
+
+[0.0.17]
+- Upgrade testing
+- App auto updates
+- Usage graphs
+
+[0.0.18]
+- Rework backups and updates
+
+[0.0.19]
+- Graphite fixes
+- Avatar and Cloudron name support
+
+[0.0.20]
+- Apptask fixes
+- Chrome related fixes
+
+[0.0.21]
+- Increase nginx hostname size to 64
+
+[0.0.22]
+- Testing the e2e tests
+
+[0.0.23]
+- Better error status page
+- Fix updater and backup progress reporting
+- New avatar set
+- Improved setup wizard
+
+[0.0.24]
+- Hotfix the ldap support
+
+[0.0.25]
+- Add support page
+- Really fix ldap issues
+
+[0.0.26]
+- Add configurePath support
+
+[0.0.27]
+- Improved log collector
+
+[0.0.28]
+- Improve app feedback
+- Restyle login page
+
+[0.0.29]
+- Update to ubuntu 15.04
+
+[0.0.30]
+- Move to docker 1.7
+
+[0.0.31]
+- WARNING: This update restarts your containers
+- System processes are prioritized over apps
+- Add ldap group support
+
+[0.0.32]
+- MySQL addon update
+
+[0.0.33]
+- Fix graphs
+- Fix MySQL 5.6 memory usage
+
+[0.0.34]
+- Correctly mark apps pending for approval
+
+[0.0.35]
+- Fix ldap admin group username
+
+[0.0.36]
+- Fix restore without backup
+- Optimize image deletion during updates
+- Add memory accounting
+- Restrict access to metadata from containers
+
+[0.0.37]
+- Prepare for Selfhosting 1. part
+- Use userData instead of provisioning calls
+
+[0.0.38]
+- Account for Ext4 reserved block when partitioning disk
+
+[0.0.39]
+- Move subdomain management to the cloudron
+
+[0.0.40]
+- Add journal limit
+- Fix reprovisioning on reboot
+- Fix subdomain management during startup
+
+[0.0.41]
+- Finally bring things to a sane state
+
+[0.0.42]
+- Parallel apptask
+
+[0.0.43]
+- Move to systemd
+
+[0.0.44]
+- Fix apptask concurrency bug
+
+[0.0.45]
+- Retry subdomain registration
+
+[0.0.46]
+- Fix app update email notification
+
+[0.0.47]
+- Ensure box code quits within 5 seconds
+
+[0.0.48]
+- Styling fixes
+- Improved session handling
+
+[0.0.49]
+- Fix app autoupdate logic
+
+[0.0.50]
+- Use domainmanagement via CaaS
+
+[0.0.51]
+- Fix memory management
+
+[0.0.52]
+- Restrict addons memory
+- Get nofication about container OOMs
+
+[0.0.53]
+- Restrict addons memory
+- Get notification about container OOMs
+- Add retry to subdomain logic
+
+[0.0.54]
+- OAuth Proxy now uses internal port forwarding
+
+[0.0.55]
+- Setup cloudron timezone based on droplet region
+
+[0.0.56]
+- Use correct timezone in updater
+
+[0.0.57]
+- Fix systemd logging issues
+
+[0.0.58]
+- Ensure backups of failed apps are retained across archival cycles
+
+[0.0.59]
+- Installer API fixes
+
+[0.0.60]
+- Do full box backup on updates
+
+[0.0.61]
+- Track update notifications to inform admin only once
+
+[0.0.62]
+- Export bind dn and password from LDAP addon
+
+[0.0.63]
+- Fix creation of TXT records
+
+[0.0.64]
+- Stop apps in a retired cloudron
+- Retry downloading application on failure
+
+[0.0.65]
+- Do not send crash mails for apps in development
+
+[0.0.66]
+- Readonly application and addon containers
+
+[0.0.67]
+- Fix email notifications
+- Fix bug when restoring from certain backups
+
+[0.0.68]
+- Update graphite image
+- Add simpleauth addon support
+
+[0.0.69]
+- Support newer manifest format
+- Fix app listing rendering in chrome
+- Fix redis backup across upgrades
+
+[0.0.70]
+- Retry app download on error
+
+[0.0.71]
+- Fix oauth and simple auth login
+
+[0.0.72]
+- Cleanup application volumes periodically
+- New application logging design
+
+[0.0.73]
+- Update SSL certificate
+
+[0.0.74]
+- Support singleUser apps
+
+[0.0.75]
+- scheduler addon
+
+[0.0.76]
+- DNS Sync fixes
+- Show warning to user when memory limit reached
+
+[0.0.77]
+- Do not set hostname in app containers
+
+[0.0.78]
+- Support custom domains
+
+[0.0.79]
+- Move SSH Port
+
+[0.0.80]
+- Use journalctl for container logs
+
+[0.1.0]
+- Wait for configuration changes before starting Cloudron
+
+[0.1.1]
+- Ensure dns config for all cloudrons
+
+[0.1.2]
+- Make email work again
+- Add DKIM keys for custom domains
+
+[0.1.3]
+- Storage backend
+
+[0.1.4]
+- CaaS Backup configuration fix
+
+[0.1.5]
+- Use correct tokens for DNS backend
+
+[0.1.6]
+- Add hook to determine the api server of the box
+- Fix crash notification
+
+[0.2.0]
+- New cloudron exec implementation
+
+[0.2.1]
+- Update to node 4.1.1
+- Fix certification installation with custom domains
+
+[0.2.2]
+- Better debug output
+- Retry more times if docker registry goes down
+
+[0.3.0]
+- Update SSH keys
+- Allow bigger manifest files
+
+[0.4.0]
+- Update to docker 1.9.0
+
+[0.4.1]
+- Fix scheduler crash
+- Crucial OAuth fixes
+
+[0.4.2]
+- Fix crash when reporting backup error
+- Allow larger manifests
+
+[0.4.3]
+- Fix cloudron exec
+
+[0.4.4]
+- Initial Lets Encrypt integration
+
+[0.4.5]
+- Fixup nginx configuration to allow dynamic certificates
+
+[0.4.6]
+- LetsEncrypt integration for custom domains
+- Rate limit crash emails
+
+[0.5.0]
+- Enable staging Lets Encrypt Integration
+
+[0.5.1]
+- Display error dialog for app installation errors
+- Enable prod Lets Encrypt Integration
+- Handle apptask crashes correctly
+
+[0.5.2]
+- Fix apphealthtask crash
+- Use cgroup fs driver instead of systemd cgroup driver in docker
+
+[0.5.3]
+- Changes for e2e testing
+
+[0.5.4]
+- Fix bug in LE server selection
+
+[0.5.5]
+- Scheduler redesign
+- Fix journalctl logging
+
+[0.5.6]
+- Prepare for selfhosting option
+
+[0.5.7]
+- Move app images off the btrfs subvolume
+
+[0.6.0]
+- Consolidate code repositories
+
+[0.6.1]
+- Use no-reply as email from address for apps in naked domains
+- Update Lets Encrypt account with owner email when available
+- Fix email templates to indicate auto update
+- Add notification UI
+
+[0.6.2]
+- Fix `cloudron exec` container to have same namespaces as app
+- Add developmentMode to manifest
+

README.md

@@ -1,17 +1,16 @@
-The Box
-=======
+Cloudron a Smart Server
+=======================
 
-Development setup
------------------
-* sudo useradd -m yellowtent
-** This dummy user is required for supervisor 'box' configs
-** Add admin-localhost as 127.0.0.1 in /etc/hosts
-** All apps will be installed as hypened-subdomains of localhost. You should add
-   hyphened-subdomains of your apps into /etc/hosts
 
-Running
--------
-* `./run.sh` - this starts up nginx to serve up the webadmin
-* `DEBUG=box:* ./app.js` - this the main box code.
-* Navigate to https://admin-localhost
 
+Selfhost Instructions
+---------------------
+
+The smart server currently relies on an AWS account with access to Route53 and S3 and is tested on DigitalOcean and EC2.
+
+First create a virtual private server with Ubuntu 15.04 and run the following commands in an ssh session to initialize the base image:
+
+```
+TODO curl from a well known released version of installer.sh
+./installer.sh <domain> <aws access key> <aws acccess secret> <backup bucket> <provider> <release sha1>
+```

app.js

@@ -1,35 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var server = require('./src/server.js'),
-    config = require('./config.js');
-
-console.log();
-console.log('==========================================');
-console.log(' Cloudron will use the following settings ');
-console.log('==========================================');
-console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : (config.LOCAL ? 'LOCAL' : 'TEST'));
-console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore token:                 ', config.token());
-console.log(' Appstore server origin:         ', config.appServerUrl());
-console.log();
-console.log('==========================================');
-console.log();
-
-server.start(function (err) {
-    if (err) {
-        console.error('Error starting server', err);
-        process.exit(1);
-    }
-
-    console.log('Server listening on port ' + config.get('port'));
-});
-
-var NOOP_CALLBACK = function () { };
-
-process.on('SIGINT', function () { server.stop(NOOP_CALLBACK); });
-process.on('SIGTERM', function () { server.stop(NOOP_CALLBACK); });

apphealthtask.js

@@ -1,135 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var appdb = require('./src/appdb.js'),
-    assert = require('assert'),
-    async = require('async'),
-    database = require('./src/database.js'),
-    DatabaseError = require('./src/databaseerror.js'),
-    debug = require('debug')('box:apphealthtask'),
-    docker = require('./src/docker.js'),
-    mailer = require('./src/mailer.js'),
-    os = require('os'),
-    superagent = require('superagent');
-
-exports = module.exports = {
-    initialize: initialize,
-    run: run
-};
-
-var FATAL_CALLBACK = function (error) {
-    if (!error) return;
-    console.error(error);
-    process.exit(2);
-};
-
-var HEALTHCHECK_INTERVAL = 30000;
-var gLastSeen = { }; // { time, emailSent }
-
-function initialize(callback) {
-    async.series([
-        database.initialize,
-        mailer.initialize
-    ], callback);
-}
-
-function setHealth(app, alive, runState, callback) {
-    assert(typeof app === 'object');
-    assert(typeof alive === 'boolean');
-    assert(typeof runState === 'string');
-    assert(typeof callback === 'function');
-
-    var healthy = true; // app is unhealthy if not alive for 2 mins
-    var now = new Date();
-
-    if (alive || !(app.id in gLastSeen)) { // give never seen apps 2 mins to come up
-        gLastSeen[app.id] = { time: now, emailSent: false };
-    } else if (Math.abs(now - gLastSeen[app.id].time) > 120 * 1000) { // not seen for 2 mins
-        debug('app %s not seen for more than 2 mins, marking as unhealthy', app.id);
-        healthy = false;
-    }
-
-    if (!healthy && !gLastSeen[app.id].emailSent) {
-        gLastSeen[app.id].emailSent = true;
-        mailer.appDied(app);
-    }
-
-    appdb.setHealth(app.id, healthy, runState, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled?
-        if (error) return callback(error);
-
-        app.healthy = healthy;
-        app.runState = runState;
-
-        callback(null);
-    });
-}
-
-
-// # TODO should probably poll from the outside network instead of the docker network?
-// callback is called with error for fatal errors and not if health check failed
-function checkAppHealth(app, callback) {
-    // only check status of installed apps. we could possibly optimize more by checking runState as well
-    if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(null);
-
-    var container = docker.getContainer(app.containerId),
-        manifest = app.manifest;
-
-    container.inspect(function (err, data) {
-        if (err || !data || !data.State) {
-            debug('Error inspecting container');
-            return setHealth(app, false, appdb.RSTATE_ERROR, callback);
-        }
-
-        if (data.State.Running !== true) {
-            debug('app %s has exited', app.id);
-            return setHealth(app, false, appdb.RSTATE_DEAD, callback);
-        }
-
-        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
-        superagent
-            .get(healthCheckUrl)
-            .timeout(HEALTHCHECK_INTERVAL)
-            .end(function (error, res) {
-
-            if (error || res.status !== 200) {
-                debug('app %s is not alive ', app.id);
-                setHealth(app, false, appdb.RSTATE_RUNNING, callback);
-            } else {
-                debug('app %s is alive', app.id);
-                setHealth(app, true, appdb.RSTATE_RUNNING, callback);
-            }
-        });
-    });
-}
-
-function processApps(callback) {
-    appdb.getAll(function (error, apps) {
-        if (error) return callback(error);
-
-        async.each(apps, checkAppHealth, function (error) {
-            if (error) console.error(error);
-            callback(null);
-        });
-    });
-}
-
-function run(callback) {
-    processApps(function (error) {
-        if (error) return callback(error);
-        setTimeout(run.bind(null, callback), HEALTHCHECK_INTERVAL);
-    });
-}
-
-if (require.main === module) {
-    initialize();
-
-    run(function (error) {
-        console.error('apphealth task exiting with error.', error);
-        process.exit(error ? 1 : 0);
-    });
-}
-

baseimage/createImage

@@ -0,0 +1,192 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+provider="digitalocean"
+installer_revision=$(git rev-parse HEAD)
+box_name=""
+server_id=""
+server_ip=""
+destroy_server="yes"
+deploy_env="dev"
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "provider:,revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --env) deploy_env="$2"; shift 2;;
+    --revision) installer_revision="$2"; shift 2;;
+    --provider) provider="$2"; shift 2;;
+    --name) box_name="$2"; destroy_server="no"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+echo "Creating image using ${provider}"
+if [[ "${provider}" == "digitalocean" ]]; then
+    if [[ "${deploy_env}" == "staging" ]]; then
+        assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
+        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
+    elif [[ "${deploy_env}" == "dev" ]]; then
+        assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
+        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
+    elif [[ "${deploy_env}" == "prod" ]]; then
+        assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
+        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
+    else
+        echo "No such env ${deploy_env}."
+        exit 1
+    fi
+
+    vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+else
+    echo "Unknown provider : ${provider}"
+    exit 1
+fi
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}"
+readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${installer_revision}")
+
+if [[ -z "${box_name}" ]]; then
+    # if you change this, change the regexp is appstore/janitor.js
+    box_name="box-${deploy_env}-${pretty_revision}-${now}" # remove slashes
+
+    # create a new server if no name given
+    if ! caas_ssh_key_id=$($vps get_ssh_key_id "caas"); then
+        echo "Could not query caas ssh key"
+        exit 1
+    fi
+    echo "Detected caas ssh key id: ${caas_ssh_key_id}"
+
+    echo "Creating Server with name [${box_name}]"
+    if ! server_id=$($vps create ${caas_ssh_key_id} ${box_name}); then
+        echo "Failed to create server"
+        exit 1
+    fi
+    echo "Created server with id: ${server_id}"
+
+    # If we run scripts overenthusiastically without the wait, setup script randomly fails
+    echo -n "Waiting 120 seconds for server creation"
+    for i in $(seq 1 24); do
+        echo -n "."
+        sleep 5
+    done
+    echo ""
+else
+    if ! server_id=$($vps get_id "${box_name}"); then
+        echo "Could not determine id from name"
+        exit 1
+    fi
+    echo "Reusing server with id: ${server_id}"
+
+    $vps power_on "${server_id}"
+fi
+
+# Query until we get an IP
+while true; do
+    echo "Trying to get the server IP"
+    if server_ip=$($vps get_ip "${server_id}"); then
+        echo "Server IP : [${server_ip}]"
+        break
+    fi
+    echo "Timedout, trying again in 10 seconds"
+    sleep 10
+done
+
+while true; do
+    echo "Trying to copy init script to server"
+    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" root@${server_ip}:.; then
+        break
+    fi
+    echo "Timedout, trying again in 30 seconds"
+    sleep 30
+done
+
+echo "Copying INFRA_VERSION"
+$scp22 "${SCRIPT_DIR}/../setup/INFRA_VERSION" root@${server_ip}:.
+
+echo "Copying box source"
+cd "${SOURCE_DIR}"
+git archive --format=tar HEAD | $ssh22 "root@${server_ip}" "cat - > /tmp/box.tar.gz"
+
+echo "Executing init script"
+if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+    echo "Init script failed"
+    exit 1
+fi
+
+echo "Copy over certs"
+cd "${SCRIPT_DIR}/../../secrets"
+blackbox_cat installer/server.crt.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/server.crt"
+blackbox_cat installer/server.key.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/server.key"
+blackbox_cat installer_ca/ca.crt.gpg  | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/ca.crt"
+
+echo "Shutting down server with id : ${server_id}"
+$ssh202 "root@${server_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail
+
+# wait 10 secs for actual shutdown
+echo "Waiting for 10 seconds for server to shutdown"
+sleep 30
+
+echo "Powering off server"
+if ! $vps power_off "${server_id}"; then
+    echo "Could not power off server"
+    exit 1
+fi
+
+snapshot_name="box-${deploy_env}-${pretty_revision}-${now}"
+echo "Snapshotting as ${snapshot_name}"
+if ! image_id=$($vps snapshot "${server_id}" "${snapshot_name}"); then
+    echo "Could not snapshot and get image id"
+    exit 1
+fi
+
+if [[ "${destroy_server}" == "yes" ]]; then
+    echo "Destroying server"
+    if ! $vps destroy "${server_id}"; then
+        echo "Could not destroy server"
+        exit 1
+    fi
+else
+    echo "Skipping server destroy"
+fi
+
+echo "Transferring image ${image_id} to other regions"
+$vps transfer_image_to_all_regions "${image_id}"
+
+echo "Done."

baseimage/digitalocean.sh

@@ -0,0 +1,240 @@
+#!/bin/bash
+
+if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then
+    echo "Script requires DIGITAL_OCEAN_TOKEN env to be set"
+    exit 1
+fi
+
+if [[ -z "${JSON}" ]]; then
+    echo "Script requires JSON env to be set to path of JSON binary"
+    exit 1
+fi
+
+readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+
+function debug() {
+    echo "$@" >&2
+}
+
+function get_ssh_key_id() {
+    id=$($CURL "https://api.digitalocean.com/v2/account/keys" \
+        | $JSON ssh_keys \
+        | $JSON -c "this.name === \"$1\"" \
+        | $JSON 0.id)
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function create_droplet() {
+    local ssh_key_id="$1"
+    local box_name="$2"
+
+    local image_region="sfo1"
+    local ubuntu_image_slug="ubuntu-15-04-x64" # id=12658446
+    local box_size="512mb"
+
+    local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
+
+    id=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id)
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function get_droplet_ip() {
+    local droplet_id="$1"
+    ip=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address")
+    [[ -z "$ip" ]] && exit 1
+    echo "$ip"
+}
+
+function get_droplet_id() {
+    local droplet_name="$1"
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    [[ -z "$id" ]] && exit 1
+    echo "$id"  
+}
+
+function power_off_droplet() {
+    local droplet_id="$1"
+    local data='{"type":"power_off"}'
+    local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions")
+    local event_id=`echo "${response}" | $JSON action.id`
+
+    if [[ -z "${event_id}" ]]; then
+        debug "Got no event id, assuming already powered off."
+        debug "Response: ${response}"
+        return
+    fi
+
+    debug "Powered off droplet. Event id: ${event_id}"
+    debug -n "Waiting for droplet to power off"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function power_on_droplet() {
+    local droplet_id="$1"
+    local data='{"type":"power_on"}'
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
+
+    debug "Powered on droplet. Event id: ${event_id}"
+
+    if [[ -z "${event_id}" ]]; then
+        debug "Got no event id, assuming already powered on"
+        return
+    fi
+
+    debug -n "Waiting for droplet to power on"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function get_image_id() {
+    local snapshot_name="$1"
+    local image_id=""
+
+    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
+
+    if [[ -n "${image_id}" ]]; then
+        echo "${image_id}"
+    fi
+}
+
+function snapshot_droplet() {
+    local droplet_id="$1"
+    local snapshot_name="$2"
+    local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}"
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
+
+    debug "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}"
+    debug -n "Waiting for snapshot to complete"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+
+    get_image_id "${snapshot_name}"
+}
+
+function destroy_droplet() {
+    local droplet_id="$1"
+    # TODO: check for 204 status
+    $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}"
+    debug "Droplet destroyed"
+    debug ""
+}
+
+function transfer_image() {
+    local image_id="$1"
+    local region_slug="$2"
+    local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}"
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id`
+    echo "${event_id}"
+}
+
+function wait_for_image_event() {
+    local image_id="$1"
+    local event_id="$2"
+
+    debug -n "Waiting for ${event_id}"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function transfer_image_to_all_regions() {
+    local image_id="$1"
+
+    xfer_events=()
+    image_regions=(ams3) ## sfo1 is where the image is created
+    for image_region in ${image_regions[@]}; do
+        xfer_event=$(transfer_image ${image_id} ${image_region})
+        echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"
+        xfer_events+=("${xfer_event}")
+        sleep 1
+    done
+
+    echo "Image transfer initiated, but they will take some time to get transferred."
+
+    for xfer_event in ${xfer_events[@]}; do
+        $vps wait_for_image_event "${image_id}" "${xfer_event}"
+    done
+}
+
+if [[ $# -lt 1 ]]; then
+    debug "<command> <params...>"
+    exit 1
+fi
+
+case $1 in
+get_ssh_key_id)
+    get_ssh_key_id "${@:2}"
+    ;;
+
+create)
+    create_droplet "${@:2}"
+    ;;
+
+get_id)
+    get_droplet_id "${@:2}"
+    ;;
+
+get_ip)
+    get_droplet_ip "${@:2}"
+    ;;
+
+power_on)
+    power_on_droplet "${@:2}"
+    ;;
+
+power_off)
+    power_off_droplet "${@:2}"
+    ;;
+
+snapshot)
+    snapshot_droplet "${@:2}"
+    ;;
+
+destroy)
+    destroy_droplet "${@:2}"
+    ;;
+
+transfer_image_to_all_regions)
+    transfer_image_to_all_regions "${@:2}"
+    ;;
+
+*)
+    echo "Unknown command $1"
+    exit 1
+esac

baseimage/initializeBaseUbuntuImage.sh

@@ -0,0 +1,308 @@
+#!/bin/bash
+
+set -euv -o pipefail
+
+readonly USER=yellowtent
+readonly USER_HOME="/home/${USER}"
+readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
+readonly INSTALLER_REVISION="$1"
+readonly SELFHOSTED=$(( $# > 1 ? 1 : 0 ))
+readonly USER_DATA_FILE="/root/user_data.img"
+readonly USER_DATA_DIR="/home/yellowtent/data"
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
+    source "${SOURCE_DIR}/INFRA_VERSION"
+else
+    echo "No INFRA_VERSION found, skip pulling docker images"
+fi
+
+if [ ${SELFHOSTED} == 0 ]; then
+    echo "!! Initializing Ubuntu image for CaaS"
+else
+    echo "!! Initializing Ubuntu image for Selfhosting"
+fi
+
+echo "==== Create User ${USER} ===="
+if ! id "${USER}"; then
+    useradd "${USER}" -m
+fi
+
+echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ==="
+
+echo "=== Prepare installer source ==="
+rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}"
+rm -rf /tmp/box && mkdir -p /tmp/box
+tar xvf /tmp/box.tar.gz -C /tmp/box && rm /tmp/box.tar.gz
+cp -rf /tmp/box/installer/* "${INSTALLER_SOURCE_DIR}"
+echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION"
+
+export DEBIAN_FRONTEND=noninteractive
+
+echo "=== Upgrade ==="
+apt-get update
+apt-get upgrade -y
+apt-get install -y curl
+
+# Setup firewall before everything. docker creates it's own chain and the -X below will remove it
+# Do NOT use iptables-persistent because it's startup ordering conflicts with docker
+echo "=== Setting up firewall ==="
+# clear tables and set default policy
+iptables -F # flush all chains
+iptables -X # delete all chains
+# default policy for filter table
+iptables -P INPUT DROP
+iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject
+iptables -P OUTPUT ACCEPT
+
+# NOTE: keep these in sync with src/apps.js validatePortBindings
+# allow ssh, http, https, ping, dns
+iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+if [ ${SELFHOSTED} == 0 ]; then
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
+else
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
+fi
+iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
+iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
+iptables -A INPUT -p udp --sport 53 -j ACCEPT
+iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+
+# loopback
+iptables -A INPUT -i lo -j ACCEPT
+iptables -A OUTPUT -o lo -j ACCEPT
+
+# prevent DoS
+# iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT
+
+# log dropped incoming. keep this at the end of all the rules
+iptables -N LOGGING # new chain
+iptables -A INPUT -j LOGGING # last rule in INPUT chain
+iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
+iptables -A LOGGING -j DROP
+
+echo "==== Install btrfs tools ==="
+apt-get -y install btrfs-tools
+
+echo "==== Install docker ===="
+# install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
+curl https://get.docker.com/builds/Linux/x86_64/docker-1.9.1 > /usr/bin/docker
+chmod +x /usr/bin/docker
+groupadd docker
+cat > /etc/systemd/system/docker.socket <<EOF
+[Unit]
+Description=Docker Socket for the API
+PartOf=docker.service
+
+[Socket]
+ListenStream=/var/run/docker.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target
+EOF
+cat > /etc/systemd/system/docker.service <<EOF
+[Unit]
+Description=Docker Application Container Engine
+After=network.target docker.socket
+Requires=docker.socket
+
+[Service]
+ExecStart=/usr/bin/docker daemon -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+echo "=== Setup btrfs docker data ==="
+fallocate -l "8192m" "${USER_DATA_FILE}" # 8gb start
+mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
+echo "${USER_DATA_FILE} ${USER_DATA_DIR} btrfs loop,nosuid 0 0" >> /etc/fstab
+mkdir -p "${USER_DATA_DIR}" && mount "${USER_DATA_FILE}"
+
+systemctl daemon-reload
+systemctl enable docker
+systemctl start docker
+
+# give docker sometime to start up and create iptables rules
+# those rules come in after docker has started, and we want to wait for them to be sure iptables-save has all of them
+sleep 10
+
+# Disable forwarding to metadata route from containers
+iptables -I FORWARD -d 169.254.169.254 -j DROP
+
+# ubuntu will restore iptables from this file automatically. this is here so that docker's chain is saved to this file
+mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4
+
+echo "=== Enable memory accounting =="
+sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
+
+# now add the user to the docker group
+usermod "${USER}" -a -G docker
+
+if [ -z $(echo "${INFRA_VERSION}") ]; then
+    echo "Skip pulling base docker images"
+else
+    echo "=== Pulling base docker images ==="
+    docker pull "${BASE_IMAGE}"
+
+    echo "=== Pulling mysql addon image ==="
+    docker pull "${MYSQL_IMAGE}"
+
+    echo "=== Pulling postgresql addon image ==="
+    docker pull "${POSTGRESQL_IMAGE}"
+
+    echo "=== Pulling redis addon image ==="
+    docker pull "${REDIS_IMAGE}"
+
+    echo "=== Pulling mongodb addon image ==="
+    docker pull "${MONGODB_IMAGE}"
+
+    echo "=== Pulling graphite docker images ==="
+    docker pull "${GRAPHITE_IMAGE}"
+
+    echo "=== Pulling mail relay ==="
+    docker pull "${MAIL_IMAGE}"
+fi
+
+echo "==== Install nginx ===="
+apt-get -y install nginx-full
+
+echo "==== Install build-essential ===="
+apt-get -y install build-essential rcconf
+
+
+echo "==== Install mysql ===="
+debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
+debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
+apt-get -y install mysql-server
+
+echo "==== Install pwgen ===="
+apt-get -y install pwgen
+
+echo "==== Install collectd ==="
+if ! apt-get install -y collectd collectd-utils; then
+    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+fi
+update-rc.d -f collectd remove
+
+# this simply makes it explicit that we run logrotate via cron. it's already part of base ubuntu
+echo "==== Install logrotate ==="
+apt-get install -y cron logrotate
+systemctl enable cron
+
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+
+echo "=== Rebuilding npm packages ==="
+cd "${INSTALLER_SOURCE_DIR}" && npm install --production
+chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
+
+echo "==== Install installer systemd script ===="
+provisionEnv="PROVISION=digitalocean"
+if [ ${SELFHOSTED} == 1 ]; then
+    provisionEnv="PROVISION=local"
+fi
+
+cat > /etc/systemd/system/cloudron-installer.service <<EOF
+[Unit]
+Description=Cloudron Installer
+
+[Service]
+Type=idle
+ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
+Environment="DEBUG=installer*,connect-lastmile" ${provisionEnv}
+KillMode=process
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Restore iptables before docker
+echo "==== Install iptables-restore systemd script ===="
+cat > /etc/systemd/system/iptables-restore.service <<EOF
+[Unit]
+Description=IPTables Restore
+Before=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/sbin/iptables-restore /etc/iptables/rules.v4
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Allocate swap files
+# https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
+echo "==== Install box-setup systemd script ===="
+cat > /etc/systemd/system/box-setup.service <<EOF
+[Unit]
+Description=Box Setup
+Before=docker.service
+After=do-resize.service
+
+[Service]
+Type=oneshot
+ExecStart="${INSTALLER_SOURCE_DIR}/systemd/box-setup.sh"
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable cloudron-installer
+systemctl enable iptables-restore
+systemctl enable box-setup
+
+# Configure systemd
+sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
+    -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
+    -i /etc/systemd/journald.conf
+
+sync
+
+# Configure time
+sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+timedatectl set-ntp 1
+timedatectl set-timezone UTC
+
+# Give user access to system logs
+apt-get -y install acl
+usermod -a -G systemd-journal ${USER}
+mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
+chown root:systemd-journal /var/log/journal
+systemctl restart systemd-journald
+setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
+
+if [ ${SELFHOSTED} == 0 ]; then
+    echo "==== Install ssh ==="
+    apt-get -y install openssh-server
+    # https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
+    sed -e 's/^#\?Port .*/Port 202/g' \
+        -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
+        -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
+        -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
+        -i /etc/ssh/sshd_config
+
+    # required so we can connect to this machine since port 22 is blocked by iptables by now
+    systemctl reload sshd
+fi

box.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
+var appHealthMonitor = require('./src/apphealthmonitor.js'),
+    async = require('async'),
+    config = require('./src/config.js'),
+    ldap = require('./src/ldap.js'),
+    oauthproxy = require('./src/oauthproxy.js'),
+    server = require('./src/server.js'),
+    simpleauth = require('./src/simpleauth.js');
+
+console.log();
+console.log('==========================================');
+console.log(' Cloudron will use the following settings ');
+console.log('==========================================');
+console.log();
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
+console.log(' Version:                        ', config.version());
+console.log(' Admin Origin:                   ', config.adminOrigin());
+console.log(' Appstore API server origin:     ', config.apiServerOrigin());
+console.log(' Appstore Web server origin:     ', config.webServerOrigin());
+console.log();
+console.log('==========================================');
+console.log();
+
+async.series([
+    server.start,
+    ldap.start,
+    simpleauth.start,
+    appHealthMonitor.start,
+    oauthproxy.start
+], function (error) {
+    if (error) {
+        console.error('Error starting server', error);
+        process.exit(1);
+    }
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});

config.js

@@ -1,143 +0,0 @@
-/* jslint node: true */
-
-'use strict';
-
-var path = require('path'),
-    fs = require('fs'),
-    safe = require('safetydance'),
-    assert = require('assert'),
-    _ = require('underscore'),
-    path = require('path'),
-    mkdirp = require('mkdirp');
-
-exports = module.exports = {
-    baseDir: baseDir,
-    get: get,
-    set: set,
-
-    // ifdefs to check environment
-    CLOUDRON: process.env.NODE_ENV === 'cloudron',
-    TEST: process.env.NODE_ENV === 'test',
-    LOCAL: process.env.NODE_ENV === 'local' || !process.env.NODE_ENV,
-
-    // convenience getters
-    appServerUrl: appServerUrl,
-    fqdn: fqdn,
-    token: token,
-    version: version,
-    isCustomDomain: isCustomDomain,
-
-    // these values are derived
-    adminOrigin: adminOrigin,
-    appFqdn: appFqdn,
-    zoneName: zoneName
-};
-
-var homeDir = process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
-
-var data = { };
-
-function baseDir() {
-    if (exports.CLOUDRON) return homeDir;
-    if (exports.TEST) return path.join(homeDir, '.yellowtenttest');
-    if (exports.LOCAL) return path.join(homeDir, '.yellowtent');
-}
-
-var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
-
-function saveSync() {
-    fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
-}
-
-(function initConfig() {
-    // setup defaults
-    if (exports.CLOUDRON) {
-        data.port = 3000;
-        data.appServerUrl = process.env.APP_SERVER_URL || null; // APP_SERVER_URL is set during bootstrap in the box's supervisor manifest
-    } else if (exports.TEST) {
-        data.port = 5454;
-        data.appServerUrl = 'http://localhost:6060'; // hock doesn't support https
-    } else if (exports.LOCAL) {
-        data.port = 3000;
-        data.appServerUrl = 'http://localhost:5050';
-    } else {
-        assert(false, 'Unknown environment. This should not happen!');
-    }
-
-    data.fqdn = 'localhost';
-
-    data.token = null;
-    data.mailServer = null;
-    data.mailUsername = null;
-    data.mailDnsRecordIds = [ ];
-    data.boxVersionsUrl = null;
-    data.version = null;
-    data.isCustomDomain = false;
-
-    if (safe.fs.existsSync(cloudronConfigFileName)) {
-        var existingData = safe.JSON.parse(safe.fs.readFileSync(cloudronConfigFileName, 'utf8'));
-        _.extend(data, existingData); // overwrite defaults with saved config
-        return;
-    }
-
-    mkdirp.sync(path.dirname(cloudronConfigFileName));
-    saveSync();
-})();
-
-// set(obj) or set(key, value)
-function set(key, value) {
-    if (typeof key === 'object') {
-        var obj = key;
-        for (var k in obj) {
-            assert(k in data, 'config.js is missing key "' + k + '"');
-            data[k] = obj[k];
-        }
-    } else {
-        assert(key in data, 'config.js is missing key "' + key + '"');
-        data[key] = value;
-    }
-    saveSync();
-}
-
-function get(key) {
-    assert(typeof key === 'string');
-
-    return safe.query(data, key);
-}
-
-function appServerUrl() {
-    return get('appServerUrl');
-}
-
-function fqdn() {
-    return get('fqdn');
-}
-
-function appFqdn(location) {
-    assert(typeof location === 'string');
-    return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
-}
-
-function adminOrigin() {
-    return 'https://' + appFqdn('admin');
-}
-
-function token() {
-    return get('token');
-}
-
-function version() {
-    return get('version');
-}
-
-function isCustomDomain() {
-    return get('isCustomDomain');
-}
-
-function zoneName() {
-    if (isCustomDomain()) return fqdn(); // the appstore sets up the custom domain as a zone
-
-    // for shared domain name, strip out the hostname
-    return fqdn().substr(fqdn().indexOf('.') + 1);
-}
-

crashnotifier.js

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var assert = require('assert'),
+    mailer = require('./src/mailer.js'),
+    safe = require('safetydance'),
+    path = require('path'),
+    util = require('util');
+
+var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
+
+function collectLogs(program, callback) {
+    assert.strictEqual(typeof program, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
+    callback(null, logs);
+}
+
+function sendCrashNotification(processName) {
+    collectLogs(processName, function (error, result) {
+        if (error) {
+            console.error('Failed to collect logs.', error);
+            result = util.format('Failed to collect logs.', error);
+        }
+
+        console.log('Sending crash notification email for', processName);
+        mailer.sendCrashNotification(processName, result);
+    });
+}
+
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
+
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendCrashNotification(processName);
+}
+
+main();

gulpfile.js

@@ -2,73 +2,163 @@
 
 'use strict';
 
-var _ejs = require('ejs'),
-    ejs = require('gulp-ejs'),
+var ejs = require('gulp-ejs'),
     gulp = require('gulp'),
     del = require('del'),
-    path = require('path'),
     concat = require('gulp-concat'),
+    uglify = require('gulp-uglify'),
+    serve = require('gulp-serve'),
+    sass = require('gulp-sass'),
     sourcemaps = require('gulp-sourcemaps'),
-    fs = require('fs');
-
-_ejs.filters.basename = function (obj) {
-    return path.basename(obj);
-};
+    minifyCSS = require('gulp-minify-css'),
+    autoprefixer = require('gulp-autoprefixer'),
+    argv = require('yargs').argv;
 
 gulp.task('3rdparty', function () {
-    return gulp.src([
-            'webadmin/src/3rdparty/**/*.js',
-            'webadmin/src/3rdparty/**/*.css',
-            'webadmin/src/3rdparty/**/*.otf',
-            'webadmin/src/3rdparty/**/*.eot',
-            'webadmin/src/3rdparty/**/*.svg',
-            'webadmin/src/3rdparty/**/*.ttf',
-            'webadmin/src/3rdparty/**/*.woff',
-            'webadmin/src/3rdparty/**/*.js'
+    gulp.src([
+        'webadmin/src/3rdparty/**/*.js',
+        'webadmin/src/3rdparty/**/*.map',
+        'webadmin/src/3rdparty/**/*.css',
+        'webadmin/src/3rdparty/**/*.otf',
+        'webadmin/src/3rdparty/**/*.eot',
+        'webadmin/src/3rdparty/**/*.svg',
+        'webadmin/src/3rdparty/**/*.ttf',
+        'webadmin/src/3rdparty/**/*.woff',
+        'webadmin/src/3rdparty/**/*.woff2'
         ])
-        .pipe(gulp.dest('webadmin/dist/3rdparty/'));
+        .pipe(gulp.dest('webadmin/dist/3rdparty/'))
+        .pipe(gulp.dest('setup/splash/website/3rdparty'));
+
+    gulp.src('node_modules/bootstrap-sass/assets/javascripts/bootstrap.min.js')
+        .pipe(gulp.dest('webadmin/dist/3rdparty/js'))
+        .pipe(gulp.dest('setup/splash/website/3rdparty/js'));
 });
 
+
+// --------------
+// JavaScript
+// --------------
+
+gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
+
+var oauth = {
+    clientId: argv.clientId || 'cid-webadmin',
+    clientSecret: argv.clientSecret || 'unused',
+    apiOrigin: argv.apiOrigin || ''
+};
+
+console.log();
+console.log('Using OAuth credentials:');
+console.log(' ClientId:     %s', oauth.clientId);
+console.log(' ClientSecret: %s', oauth.clientSecret);
+console.log(' Cloudron API: %s', oauth.apiOrigin || 'default');
+console.log();
+
 gulp.task('js-index', function () {
-    return gulp.src(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'])
+    gulp.src([
+        'webadmin/src/js/index.js',
+        'webadmin/src/js/client.js',
+        'webadmin/src/js/appstore.js',
+        'webadmin/src/js/main.js',
+        'webadmin/src/views/*.js'
+        ])
+        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
         .pipe(sourcemaps.init())
-        .pipe(concat('index.js'))
+        .pipe(concat('index.js', { newLine: ';' }))
+        .pipe(uglify())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
 gulp.task('js-setup', function () {
-    return gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
+    gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
+        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
         .pipe(sourcemaps.init())
-        .pipe(concat('setup.js'))
+        .pipe(concat('setup.js', { newLine: ';' }))
+        .pipe(uglify())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('js', ['js-index', 'js-setup'], function () {});
-
-gulp.task('htmlViews', function () {
-    return gulp.src('webadmin/src/views/*.html')
-        .pipe(gulp.dest('webadmin/dist/views'));
+gulp.task('js-error', function () {
+    gulp.src(['webadmin/src/js/error.js'])
+        .pipe(sourcemaps.init())
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('html_templates', function () {
-    var config = JSON.parse(fs.readFileSync('./webadmin/deploymentConfig.json'));
-
-    return gulp.src('webadmin/src/*.ejs')
-        .pipe(ejs(config, { ext: '.html' }))
-        .pipe(gulp.dest('webadmin/dist'));
+gulp.task('js-update', function () {
+    gulp.src(['webadmin/src/js/update.js'])
+        .pipe(sourcemaps.init())
+        .pipe(uglify())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist/js'))
+        .pipe(gulp.dest('setup/splash/website/js'));
 });
 
-gulp.task('html', ['html_templates', 'htmlViews'], function () {
-    return gulp.src('webadmin/src/*.html')
-        .pipe(gulp.dest('webadmin/dist'));
+
+// --------------
+// HTML
+// --------------
+
+gulp.task('html', ['html-views', 'html-update', 'html-templates'], function () {
+    return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
 });
 
-gulp.task('clean', function (callback) {
-    del(['webadmin/dist'], callback);
+gulp.task('html-update', function () {
+    return gulp.src(['webadmin/src/update.html']).pipe(gulp.dest('setup/splash/website'));
 });
 
-gulp.task('default', ['clean'], function () {
-    gulp.start('html', 'js', '3rdparty');
+gulp.task('html-views', function () {
+    return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
 });
+
+gulp.task('html-templates', function () {
+    return gulp.src('webadmin/src/templates/**/*.html').pipe(gulp.dest('webadmin/dist/templates'));
+});
+
+// --------------
+// CSS
+// --------------
+
+gulp.task('css', function () {
+    return gulp.src('webadmin/src/*.scss')
+        .pipe(sourcemaps.init())
+        .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
+        .pipe(autoprefixer())
+        .pipe(minifyCSS())
+        .pipe(sourcemaps.write())
+        .pipe(gulp.dest('webadmin/dist'))
+        .pipe(gulp.dest('setup/splash/website'));
+});
+
+gulp.task('images', function () {
+    return gulp.src('webadmin/src/img/**')
+        .pipe(gulp.dest('webadmin/dist/img'));
+});
+
+// --------------
+// Utilities
+// --------------
+
+gulp.task('watch', ['default'], function () {
+    gulp.watch(['webadmin/src/*.scss'], ['css']);
+    gulp.watch(['webadmin/src/img/*'], ['images']);
+    gulp.watch(['webadmin/src/**/*.html'], ['html']);
+    gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
+    gulp.watch(['webadmin/src/templates/*.html'], ['html-templates']);
+    gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
+    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
+    gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
+    gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
+    gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);
+});
+
+gulp.task('clean', function () {
+    del.sync(['webadmin/dist', 'setup/splash/website']);
+});
+
+gulp.task('default', ['clean', 'html', 'js', '3rdparty', 'images', 'css'], function () {});
+
+gulp.task('develop', ['watch'], serve({ root: 'webadmin/dist', port: 4000 }));

installer.sh

@@ -0,0 +1,159 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+echo ""
+echo "======== Cloudron Installer ========"
+echo ""
+
+if [ $# -lt 4 ]; then
+    echo "Usage: ./installer.sh <fqdn> <aws key id> <aws key secret> <bucket> <provider> <revision>"
+    exit 1
+fi
+
+# commandline arguments
+readonly fqdn="${1}"
+readonly aws_access_key_id="${2}"
+readonly aws_access_key_secret="${3}"
+readonly aws_backup_bucket="${4}"
+readonly provider="${5}"
+readonly revision="${6}"
+
+# environment specific urls
+readonly api_server_origin="https://api.dev.cloudron.io"
+readonly web_server_origin="https://dev.cloudron.io"
+readonly release_bucket_url="https://s3.amazonaws.com/dev-cloudron-releases"
+readonly versions_url="https://s3.amazonaws.com/dev-cloudron-releases/versions.json"
+readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
+
+# runtime consts
+readonly installer_code_file="/tmp/box.tar.gz"
+readonly installer_tmp_dir="/tmp/box"
+readonly cert_folder="/tmp/certificates"
+
+# check for fqdn in /ets/hosts
+echo "[INFO] checking for hostname entry"
+readonly hostentry_found=$(grep "${fqdn}" /etc/hosts || true)
+if [[ -z $hostentry_found ]]; then
+    echo "[WARNING] No entry for ${fqdn} found in /etc/hosts"
+    echo "Adding an entry ..."
+
+    cat >> /etc/hosts <<EOF
+
+# The following line was added by the Cloudron installer script
+127.0.1.1 ${fqdn} ${fqdn}
+EOF
+else
+    echo "Valid hostname entry found in /etc/hosts"
+fi
+echo ""
+
+echo "[INFO] ensure minimal dependencies ..."
+apt-get update
+apt-get install -y curl
+echo ""
+
+echo "[INFO] Generating certificates ..."
+rm -rf "${cert_folder}"
+mkdir -p "${cert_folder}"
+
+cat > "${cert_folder}/CONFIG" <<EOF
+[ req ]
+default_bits           = 1024
+default_keyfile        = keyfile.pem
+distinguished_name     = req_distinguished_name
+prompt                 = no
+req_extensions         = v3_req
+
+[ req_distinguished_name ]
+C                      = DE
+ST                     = Berlin
+L                      = Berlin
+O                      = Cloudron UG
+OU                     = Cloudron
+CN                     = ${fqdn}
+emailAddress           = cert@cloudron.io
+
+[ v3_req ]
+# Extensions to add to a certificate request
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = ${fqdn}
+DNS.2 = *.${fqdn}
+EOF
+
+# generate cert files
+openssl genrsa 2048 > "${cert_folder}/host.key"
+openssl req -new -out "${cert_folder}/host.csr" -key "${cert_folder}/host.key" -config "${cert_folder}/CONFIG"
+openssl x509 -req -days 3650 -in "${cert_folder}/host.csr" -signkey "${cert_folder}/host.key" -out "${cert_folder}/host.cert" -extensions v3_req -extfile "${cert_folder}/CONFIG"
+
+# make them json compatible, by collapsing to one line
+tls_cert=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.cert")
+tls_key=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.key")
+echo ""
+
+echo "[INFO] Fetching installer code ..."
+curl "${installer_code_url}" -o "${installer_code_file}"
+echo ""
+
+echo "[INFO] Extracting installer code to ${installer_tmp_dir} ..."
+rm -rf "${installer_tmp_dir}" && mkdir -p "${installer_tmp_dir}"
+tar xvf "${installer_code_file}" -C "${installer_tmp_dir}"
+echo ""
+
+echo "Creating initial provisioning config ..."
+cat > /root/provision.json <<EOF
+{
+    "sourceTarballUrl": "",
+    "data": {
+        "apiServerOrigin": "${api_server_origin}",
+        "webServerOrigin": "${web_server_origin}",
+        "fqdn": "${fqdn}",
+        "token": "",
+        "isCustomDomain": true,
+        "boxVersionsUrl": "${versions_url}",
+        "version": "",
+        "tlsCert": "${tls_cert}",
+        "tlsKey": "${tls_key}",
+        "provider": "${provider}",
+        "backupConfig": {
+            "provider": "s3",
+            "accessKeyId": "${aws_access_key_id}",
+            "secretAccessKey": "${aws_access_key_secret}",
+            "bucket": "${aws_backup_bucket}",
+            "prefix": "backups"
+        },
+        "dnsConfig": {
+            "provider": "route53",
+            "accessKeyId": "${aws_access_key_id}",
+            "secretAccessKey": "${aws_access_key_secret}"
+        },
+        "tlsConfig": {
+            "provider": "letsencrypt-dev"
+        }
+    }
+}
+EOF
+
+echo "[INFO] Running Ubuntu initializing script ..."
+/bin/bash "${installer_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${revision}" selfhosting
+echo ""
+
+echo "[INFO] Reloading systemd daemon ..."
+systemctl daemon-reload
+echo ""
+
+echo "[INFO] Restart docker ..."
+systemctl restart docker
+echo ""
+
+echo "[FINISHED] Now starting Cloudron init jobs ..."
+systemctl start box-setup
+
+# TODO this is only for convenience we should probably just let the user do a restart
+sleep 5 && sync
+systemctl start cloudron-installer
+journalctl -u cloudron-installer.service -f

installer/npm-shrinkwrap.json

@@ -0,0 +1,516 @@
+{
+  "name": "installer",
+  "version": "0.0.1",
+  "dependencies": {
+    "async": {
+      "version": "1.5.0",
+      "from": "async@>=1.5.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz"
+    },
+    "body-parser": {
+      "version": "1.14.1",
+      "from": "body-parser@>=1.12.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
+      "dependencies": {
+        "bytes": {
+          "version": "2.1.0",
+          "from": "bytes@2.1.0",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz"
+        },
+        "content-type": {
+          "version": "1.0.1",
+          "from": "content-type@>=1.0.1 <1.1.0",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
+        },
+        "depd": {
+          "version": "1.1.0",
+          "from": "depd@>=1.1.0 <1.2.0",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
+        },
+        "http-errors": {
+          "version": "1.3.1",
+          "from": "http-errors@>=1.3.1 <1.4.0",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+          "dependencies": {
+            "inherits": {
+              "version": "2.0.1",
+              "from": "inherits@>=2.0.1 <2.1.0",
+              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+            },
+            "statuses": {
+              "version": "1.2.1",
+              "from": "statuses@>=1.0.0 <2.0.0",
+              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+            }
+          }
+        },
+        "iconv-lite": {
+          "version": "0.4.12",
+          "from": "iconv-lite@0.4.12",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "on-finished@>=2.3.0 <2.4.0",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "ee-first@1.1.1",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "qs": {
+          "version": "5.1.0",
+          "from": "qs@5.1.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz"
+        },
+        "raw-body": {
+          "version": "2.1.4",
+          "from": "raw-body@>=2.1.4 <2.2.0",
+          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
+          "dependencies": {
+            "unpipe": {
+              "version": "1.0.0",
+              "from": "unpipe@1.0.0",
+              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+            }
+          }
+        },
+        "type-is": {
+          "version": "1.6.9",
+          "from": "type-is@>=1.6.9 <1.7.0",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "dependencies": {
+            "media-typer": {
+              "version": "0.3.0",
+              "from": "media-typer@0.3.0",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "mime-types@>=2.1.7 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "mime-db@>=1.19.0 <1.20.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "connect-lastmile": {
+      "version": "0.0.13",
+      "from": "connect-lastmile@0.0.13",
+      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
+      "dependencies": {
+        "debug": {
+          "version": "2.1.3",
+          "from": "debug@>=2.1.0 <2.2.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
+          "dependencies": {
+            "ms": {
+              "version": "0.7.0",
+              "from": "ms@0.7.0",
+              "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
+            }
+          }
+        }
+      }
+    },
+    "debug": {
+      "version": "2.2.0",
+      "from": "debug@>=2.1.1 <3.0.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "dependencies": {
+        "ms": {
+          "version": "0.7.1",
+          "from": "ms@0.7.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        }
+      }
+    },
+    "express": {
+      "version": "4.13.3",
+      "from": "express@>=4.11.2 <5.0.0",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
+      "dependencies": {
+        "accepts": {
+          "version": "1.2.13",
+          "from": "accepts@>=1.2.12 <1.3.0",
+          "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
+          "dependencies": {
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "mime-types@>=2.1.6 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "mime-db@>=1.19.0 <1.20.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            },
+            "negotiator": {
+              "version": "0.5.3",
+              "from": "negotiator@0.5.3",
+              "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
+            }
+          }
+        },
+        "array-flatten": {
+          "version": "1.1.1",
+          "from": "array-flatten@1.1.1",
+          "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
+        },
+        "content-disposition": {
+          "version": "0.5.0",
+          "from": "content-disposition@0.5.0",
+          "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz"
+        },
+        "content-type": {
+          "version": "1.0.1",
+          "from": "content-type@>=1.0.1 <1.1.0",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
+        },
+        "cookie": {
+          "version": "0.1.3",
+          "from": "cookie@0.1.3",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz"
+        },
+        "cookie-signature": {
+          "version": "1.0.6",
+          "from": "cookie-signature@1.0.6",
+          "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
+        },
+        "depd": {
+          "version": "1.0.1",
+          "from": "depd@>=1.0.1 <1.1.0",
+          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+        },
+        "escape-html": {
+          "version": "1.0.2",
+          "from": "escape-html@1.0.2",
+          "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz"
+        },
+        "etag": {
+          "version": "1.7.0",
+          "from": "etag@>=1.7.0 <1.8.0",
+          "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
+        },
+        "finalhandler": {
+          "version": "0.4.0",
+          "from": "finalhandler@0.4.0",
+          "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
+          "dependencies": {
+            "unpipe": {
+              "version": "1.0.0",
+              "from": "unpipe@>=1.0.0 <1.1.0",
+              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+            }
+          }
+        },
+        "fresh": {
+          "version": "0.3.0",
+          "from": "fresh@0.3.0",
+          "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
+        },
+        "merge-descriptors": {
+          "version": "1.0.0",
+          "from": "merge-descriptors@1.0.0",
+          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
+        },
+        "methods": {
+          "version": "1.1.1",
+          "from": "methods@>=1.1.1 <1.2.0",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "on-finished@>=2.3.0 <2.4.0",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "ee-first@1.1.1",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "parseurl": {
+          "version": "1.3.0",
+          "from": "parseurl@>=1.3.0 <1.4.0",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
+        },
+        "path-to-regexp": {
+          "version": "0.1.7",
+          "from": "path-to-regexp@0.1.7",
+          "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
+        },
+        "proxy-addr": {
+          "version": "1.0.8",
+          "from": "proxy-addr@>=1.0.8 <1.1.0",
+          "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
+          "dependencies": {
+            "forwarded": {
+              "version": "0.1.0",
+              "from": "forwarded@>=0.1.0 <0.2.0",
+              "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz"
+            },
+            "ipaddr.js": {
+              "version": "1.0.1",
+              "from": "ipaddr.js@1.0.1",
+              "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz"
+            }
+          }
+        },
+        "qs": {
+          "version": "4.0.0",
+          "from": "qs@4.0.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
+        },
+        "range-parser": {
+          "version": "1.0.3",
+          "from": "range-parser@>=1.0.2 <1.1.0",
+          "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
+        },
+        "send": {
+          "version": "0.13.0",
+          "from": "send@0.13.0",
+          "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
+          "dependencies": {
+            "destroy": {
+              "version": "1.0.3",
+              "from": "destroy@1.0.3",
+              "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz"
+            },
+            "http-errors": {
+              "version": "1.3.1",
+              "from": "http-errors@>=1.3.1 <1.4.0",
+              "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+              "dependencies": {
+                "inherits": {
+                  "version": "2.0.1",
+                  "from": "inherits@>=2.0.1 <2.1.0",
+                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.3.4",
+              "from": "mime@1.3.4",
+              "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
+            },
+            "ms": {
+              "version": "0.7.1",
+              "from": "ms@0.7.1",
+              "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+            },
+            "statuses": {
+              "version": "1.2.1",
+              "from": "statuses@>=1.2.1 <1.3.0",
+              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+            }
+          }
+        },
+        "serve-static": {
+          "version": "1.10.0",
+          "from": "serve-static@>=1.10.0 <1.11.0",
+          "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
+        },
+        "type-is": {
+          "version": "1.6.9",
+          "from": "type-is@>=1.6.9 <1.7.0",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "dependencies": {
+            "media-typer": {
+              "version": "0.3.0",
+              "from": "media-typer@0.3.0",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "mime-types@>=2.1.6 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "mime-db@>=1.19.0 <1.20.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            }
+          }
+        },
+        "utils-merge": {
+          "version": "1.0.0",
+          "from": "utils-merge@1.0.0",
+          "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
+        },
+        "vary": {
+          "version": "1.0.1",
+          "from": "vary@>=1.0.1 <1.1.0",
+          "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
+        }
+      }
+    },
+    "json": {
+      "version": "9.0.3",
+      "from": "json@>=9.0.3 <10.0.0",
+      "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz"
+    },
+    "morgan": {
+      "version": "1.6.1",
+      "from": "morgan@>=1.5.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
+      "dependencies": {
+        "basic-auth": {
+          "version": "1.0.3",
+          "from": "basic-auth@>=1.0.3 <1.1.0",
+          "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz"
+        },
+        "depd": {
+          "version": "1.0.1",
+          "from": "depd@>=1.0.1 <1.1.0",
+          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "on-finished@>=2.3.0 <2.4.0",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "ee-first@1.1.1",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "on-headers": {
+          "version": "1.0.1",
+          "from": "on-headers@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz"
+        }
+      }
+    },
+    "proxy-middleware": {
+      "version": "0.15.0",
+      "from": "proxy-middleware@>=0.15.0 <0.16.0",
+      "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz"
+    },
+    "safetydance": {
+      "version": "0.0.19",
+      "from": "safetydance@0.0.19",
+      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
+    },
+    "semver": {
+      "version": "5.1.0",
+      "from": "semver@>=5.1.0 <6.0.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
+    },
+    "superagent": {
+      "version": "0.21.0",
+      "from": "superagent@>=0.21.0 <0.22.0",
+      "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
+      "dependencies": {
+        "component-emitter": {
+          "version": "1.1.2",
+          "from": "component-emitter@1.1.2",
+          "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz"
+        },
+        "cookiejar": {
+          "version": "2.0.1",
+          "from": "cookiejar@2.0.1",
+          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz"
+        },
+        "extend": {
+          "version": "1.2.1",
+          "from": "extend@>=1.2.1 <1.3.0",
+          "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz"
+        },
+        "form-data": {
+          "version": "0.1.3",
+          "from": "form-data@0.1.3",
+          "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
+          "dependencies": {
+            "async": {
+              "version": "0.9.2",
+              "from": "async@>=0.9.0 <0.10.0",
+              "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
+            },
+            "combined-stream": {
+              "version": "0.0.7",
+              "from": "combined-stream@>=0.0.4 <0.1.0",
+              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
+              "dependencies": {
+                "delayed-stream": {
+                  "version": "0.0.5",
+                  "from": "delayed-stream@0.0.5",
+                  "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
+                }
+              }
+            }
+          }
+        },
+        "formidable": {
+          "version": "1.0.14",
+          "from": "formidable@1.0.14",
+          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz"
+        },
+        "methods": {
+          "version": "1.0.1",
+          "from": "methods@1.0.1",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz"
+        },
+        "mime": {
+          "version": "1.2.11",
+          "from": "mime@1.2.11",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
+        },
+        "qs": {
+          "version": "1.2.0",
+          "from": "qs@1.2.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz"
+        },
+        "readable-stream": {
+          "version": "1.0.27-1",
+          "from": "readable-stream@1.0.27-1",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
+          "dependencies": {
+            "core-util-is": {
+              "version": "1.0.1",
+              "from": "core-util-is@>=1.0.0 <1.1.0",
+              "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
+            },
+            "inherits": {
+              "version": "2.0.1",
+              "from": "inherits@>=2.0.1 <2.1.0",
+              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+            },
+            "isarray": {
+              "version": "0.0.1",
+              "from": "isarray@0.0.1",
+              "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
+            },
+            "string_decoder": {
+              "version": "0.10.31",
+              "from": "string_decoder@>=0.10.0 <0.11.0",
+              "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+            }
+          }
+        },
+        "reduce-component": {
+          "version": "1.0.1",
+          "from": "reduce-component@1.0.1",
+          "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
+        }
+      }
+    }
+  }
+}

installer/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "installer",
+  "description": "Cloudron Installer",
+  "version": "0.0.1",
+  "private": "true",
+  "author": {
+    "name": "Cloudron authors"
+  },
+  "repository": {
+    "type": "git"
+  },
+  "engines": [
+    "node >=4.0.0 <=4.1.1"
+  ],
+  "dependencies": {
+    "async": "^1.5.0",
+    "body-parser": "^1.12.0",
+    "connect-lastmile": "0.0.13",
+    "debug": "^2.1.1",
+    "express": "^4.11.2",
+    "json": "^9.0.3",
+    "morgan": "^1.5.1",
+    "proxy-middleware": "^0.15.0",
+    "safetydance": "0.0.19",
+    "semver": "^5.1.0",
+    "superagent": "^0.21.0"
+  },
+  "devDependencies": {
+    "colors": "^1.1.2",
+    "commander": "^2.8.1",
+    "expect.js": "^0.3.1",
+    "istanbul": "^0.3.5",
+    "lodash": "^3.2.0",
+    "mocha": "^2.1.0",
+    "nock": "^0.59.1",
+    "sleep": "^3.0.0",
+    "superagent-sync": "^0.2.0",
+    "supererror": "^0.7.0",
+    "yesno": "0.0.1"
+  },
+  "scripts": {
+    "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test",
+    "precommit": "/bin/true",
+    "prepush": "npm test",
+    "postmerge": "/bin/true"
+  }
+}

installer/src/installer.js

@@ -0,0 +1,128 @@
+/* jslint node: true */
+
+'use strict';
+
+var assert = require('assert'),
+    child_process = require('child_process'),
+    debug = require('debug')('installer:installer'),
+    path = require('path'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    superagent = require('superagent'),
+    util = require('util');
+
+exports = module.exports = {
+    InstallerError: InstallerError,
+
+    provision: provision,
+    retire: retire,
+
+    _ensureVersion: ensureVersion
+};
+
+var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'),
+    RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh'),
+    SUDO = '/usr/bin/sudo';
+
+function InstallerError(reason, info) {
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info);
+}
+util.inherits(InstallerError, Error);
+InstallerError.INTERNAL_ERROR = 1;
+InstallerError.ALREADY_PROVISIONED = 2;
+
+function spawn(tag, cmd, args, callback) {
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof cmd, 'string');
+    assert(util.isArray(args));
+    assert.strictEqual(typeof callback, 'function');
+
+    var cp = child_process.spawn(cmd, args, { timeout: 0 });
+    cp.stdout.setEncoding('utf8');
+    cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); });
+    cp.stderr.setEncoding('utf8');
+    cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); });
+
+    cp.on('error', function (error) {
+        debug('%s : child process errored %s', tag, error.message);
+        callback(error);
+    });
+
+    cp.on('exit', function (code, signal) {
+        debug('%s : child process exited. code: %d signal: %d', tag, code, signal);
+        if (signal) return callback(new Error('Exited with signal ' + signal));
+        if (code !== 0) return callback(new Error('Exited with code ' + code));
+
+        callback(null);
+    });
+}
+
+function retire(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var pargs = [ RETIRE_CMD ];
+    pargs.push('--data', JSON.stringify(args.data));
+
+    debug('retire: calling with args %j', pargs);
+
+    if (process.env.NODE_ENV === 'test') return callback(null);
+
+    // sudo is required for retire()
+    spawn('retire', SUDO, pargs, callback);
+}
+
+function ensureVersion(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!args.data || !args.data.boxVersionsUrl) return callback(new Error('No boxVersionsUrl specified'));
+
+    if (args.sourceTarballUrl) return callback(null, args);
+
+    superagent.get(args.data.boxVersionsUrl).end(function (error, result) {
+        if (error && !error.response) return callback(error);
+        if (result.statusCode !== 200) return callback(new Error(util.format('Bad status: %s %s', result.statusCode, result.text)));
+
+        var versions = safe.JSON.parse(result.text);
+
+        if (!versions || typeof versions !== 'object') return callback(new Error('versions is not in valid format:' + safe.error));
+
+        var latestVersion = Object.keys(versions).sort(semver.compare).pop();
+        debug('ensureVersion: Latest version is %s etag:%s', latestVersion, result.header['etag']);
+
+        if (!versions[latestVersion]) return callback(new Error('No version available'));
+        if (!versions[latestVersion].sourceTarballUrl) return callback(new Error('No sourceTarballUrl specified'));
+
+        args.sourceTarballUrl = versions[latestVersion].sourceTarballUrl;
+        args.data.version = latestVersion;
+
+        callback(null, args);
+    });
+}
+
+function provision(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (process.env.NODE_ENV === 'test') return callback(null);
+
+    ensureVersion(args, function (error, result) {
+        if (error) return callback(error);
+
+        var pargs = [ INSTALLER_CMD ];
+        pargs.push('--sourcetarballurl', result.sourceTarballUrl);
+        pargs.push('--data', JSON.stringify(result.data));
+
+        debug('provision: calling with args %j', pargs);
+
+        // sudo is required for update()
+        spawn('provision', SUDO, pargs, callback);
+    });
+}
+

installer/src/scripts/installer.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly BOX_SRC_DIR=/home/yellowtent/box
+readonly DATA_DIR=/home/yellowtent/data
+
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly json="${script_dir}/../../node_modules/.bin/json"
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 180"
+
+readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
+
+# create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@
+(echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh
+chmod +x /home/yellowtent/provision.sh
+
+arg_source_tarball_url=""
+arg_data=""
+
+args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --sourcetarballurl) arg_source_tarball_url="$2";;
+    --data) arg_data="$2";;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+
+    shift 2
+done
+
+box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
+echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}"
+
+while true; do
+    if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi
+    echo "Failed to download source tarball, trying again"
+    sleep 5
+done
+while true; do
+    # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
+    if cd "${box_src_tmp_dir}" && npm rebuild; then break; fi
+    echo "Failed to rebuild, trying again"
+    sleep 5
+done
+
+if [[ "${is_update}" == "yes" ]]; then
+    echo "Setting up update splash screen"
+    "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code
+    ${BOX_SRC_DIR}/setup/stop.sh # stop the old code
+fi
+
+# switch the codes
+rm -rf "${BOX_SRC_DIR}"
+mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}"
+chown -R yellowtent.yellowtent "${BOX_SRC_DIR}"
+
+# create a start file for testing. %q escapes args
+(echo -e "#!/bin/bash\n"; printf "%q " "${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}") > /home/yellowtent/setup_start.sh
+chmod +x /home/yellowtent/setup_start.sh
+
+echo "Calling box setup script"
+"${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}"
+

installer/src/scripts/retire.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# This script is called once at the end of a cloudrons lifetime
+
+set -eu -o pipefail
+
+readonly BOX_SRC_DIR=/home/yellowtent/box
+
+arg_data=""
+
+args=$(getopt -o "" -l "data:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --data) arg_data="$2";;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+
+    shift 2
+done
+
+echo "Setting up splash screen"
+"${BOX_SRC_DIR}/setup/splashpage.sh" --retire --data "${arg_data}" # show splash
+"${BOX_SRC_DIR}/setup/stop.sh" # stop the cloudron code
+
+systemctl stop docker # stop the apps
+systemctl stop cloudron-installer # stop the installer
+

installer/src/server.js

@@ -0,0 +1,213 @@
+#!/usr/bin/env node
+
+/* jslint node: true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    debug = require('debug')('installer:server'),
+    express = require('express'),
+    fs = require('fs'),
+    http = require('http'),
+    HttpError = require('connect-lastmile').HttpError,
+    https = require('https'),
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    installer = require('./installer.js'),
+    json = require('body-parser').json,
+    lastMile = require('connect-lastmile'),
+    morgan = require('morgan'),
+    path = require('path'),
+    superagent = require('superagent');
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var PROVISION_CONFIG_FILE = '/root/provision.json';
+var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
+
+var gHttpsServer = null, // provision server; used for install/restore
+    gHttpServer = null; // update server; used for updates
+
+function provisionDigitalOcean(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
+        if (error || result.statusCode !== 200) {
+            console.error('Error getting metadata', error);
+            return callback(new Error('Error getting metadata'));
+        }
+
+        var userData = JSON.parse(result.body.user_data);
+
+        installer.provision(userData, callback);
+    });
+}
+
+function provisionLocal(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    if (!fs.existsSync(PROVISION_CONFIG_FILE)) {
+        console.error('No provisioning data found at %s', PROVISION_CONFIG_FILE);
+        return callback(new Error('No provisioning data found'));
+    }
+
+    var userData = require(PROVISION_CONFIG_FILE);
+
+    installer.provision(userData, callback);
+}
+
+function update(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.sourceTarballUrl || typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided'));
+    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
+
+    debug('provision: received from box %j', req.body);
+
+    installer.provision(req.body, function (error) {
+        if (error) console.error(error);
+    });
+
+    next(new HttpSuccess(202, { }));
+}
+
+function retire(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
+
+    if (typeof req.body.data.tlsCert !== 'string') console.error('No TLS cert provided');
+    if (typeof req.body.data.tlsKey !== 'string') console.error('No TLS key provided');
+
+    debug('retire: received from appstore %j', req.body);
+
+    installer.retire(req.body, function (error) {
+        if (error) console.error(error);
+    });
+
+    next(new HttpSuccess(202, {}));
+}
+
+function startUpdateServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting update server');
+
+    var app = express();
+
+    var router = new express.Router();
+
+    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
+
+    app.use(json({ strict: true }))
+       .use(router)
+       .use(lastMile());
+
+    router.post('/api/v1/installer/update', update);
+
+    gHttpServer = http.createServer(app);
+    gHttpServer.on('error', console.error);
+
+    gHttpServer.listen(2020, '127.0.0.1', callback);
+}
+
+function startProvisionServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting provision server');
+
+    var app = express();
+
+    var router = new express.Router();
+
+    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
+
+    app.use(json({ strict: true }))
+       .use(router)
+       .use(lastMile());
+
+    router.post('/api/v1/installer/retire', retire);
+
+    var caPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? 'test/certs' : 'certs');
+    var certPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? 'test/certs' : 'certs');
+
+    var options = {
+        key: fs.readFileSync(path.join(certPath, 'server.key')),
+        cert: fs.readFileSync(path.join(certPath, 'server.crt')),
+        ca: fs.readFileSync(path.join(caPath, 'ca.crt')),
+
+        // request cert from client and only allow from our CA
+        requestCert: true,
+        rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0' // this is set in the tests
+    };
+
+    gHttpsServer = https.createServer(options, app);
+    gHttpsServer.on('error', console.error);
+
+    gHttpsServer.listen(process.env.NODE_ENV === 'test' ? 4443 : 886, '0.0.0.0', callback);
+}
+
+function stopProvisionServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Stopping provision server');
+
+    if (!gHttpsServer) return callback(null);
+
+    gHttpsServer.close(callback);
+    gHttpsServer = null;
+}
+
+function stopUpdateServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Stopping update server');
+
+    if (!gHttpServer) return callback(null);
+
+    gHttpServer.close(callback);
+    gHttpServer = null;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var actions;
+
+    if (process.env.PROVISION === 'local') {
+        debug('Starting Installer in selfhost mode');
+
+        actions = [
+            startUpdateServer,
+            provisionLocal
+        ];
+    } else {    // current fallback, should be 'digitalocean' eventually, see initializeBaseUbuntuImage.sh
+        debug('Starting Installer in managed mode');
+
+        actions = [
+            startUpdateServer,
+            startProvisionServer,
+            provisionDigitalOcean
+        ];
+    }
+
+    async.series(actions, callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        stopUpdateServer,
+        stopProvisionServer
+    ], callback);
+}
+
+if (require.main === module) {
+    start(function (error) {
+        if (error) console.error(error);
+    });
+}

installer/src/test/certs/ca.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIID9zCCAt+gAwIBAgIJAMPL81PAySGAMA0GCSqGSIb3DQEBBQUAMFoxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIEwJDQTELMAkGA1UEBxMCU0MxFTATBgNVBAoTDENsb3Vk
+cm9uIEluYzEaMBgGA1UEAxMRSW5zdGFsbCBTZXJ2ZXIgQ0EwHhcNMTUwMTE2MDEy
+NDM2WhcNMTYwMTE2MDEyNDM2WjBaMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0Ex
+CzAJBgNVBAcTAlNDMRUwEwYDVQQKEwxDbG91ZHJvbiBJbmMxGjAYBgNVBAMTEUlu
+c3RhbGwgU2VydmVyIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+31TkOEC3JXtieHiZgM5qWw771rV2JEDKs1C68+n/OmKrp3zAQV08A+w/KVurn1P9
+gZlYF+CBRVZDV8lYbWzc6PgMPWEDHHV72FS5Kq6ZyikB+r5OQJ8qU61y840h6ZCD
+MEYr6N9qXm9wSApJBQ/key/pg7+95B2CFYRrg5NVstIYqpJ1lyxCMFTrjYAmteOB
+Bi/4GPApu9Tj0ifTMbZFGTPtWm/yhCZ6Anm6w+ok9tDMpPC6kRgUJ3B4HY75D9dV
+aWSls9jdZw4JU1jIFlAdUjhGEEmHWOzAD8vBjvuBqcf9NQwvieWG5tDYfZ6DYRC2
+/aG1C5UWhFLDv2/F+56k3wIDAQABo4G/MIG8MB0GA1UdDgQWBBQ088hd2sIIqVtw
+xJeAkCORdclFRjCBjAYDVR0jBIGEMIGBgBQ088hd2sIIqVtwxJeAkCORdclFRqFe
+pFwwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQswCQYDVQQHEwJTQzEVMBMG
+A1UEChMMQ2xvdWRyb24gSW5jMRowGAYDVQQDExFJbnN0YWxsIFNlcnZlciBDQYIJ
+AMPL81PAySGAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAJcW+Wmz
+/o0JBC2WsMjUjxVrzOiu9bdKQ1yn83Zcv74zEfmWfJotVOK1oKsTyOZfTvvWrpLc
+GXXhh4oXWsNnFII3uJyZIY3v/DoE0pa7TCZhLYFbL2kEaC5rTwe/+VScHy5ROOiu
++gnzOU3MyrcMTT0v4qcT0NlkIptRdvIYNpqfXO6vG9sMp4C/NwWhl/IfHkIAv0eH
+l3HTr8wxgldCjxbnJgYkyUcWAmLi2YEXKCEPWmsfqp3Z+Ng1M+A9OKjJLHWowl9X
+4arvn6WaUbZjRxxjvK199If1R6KWwD6YQ9cKH4Ex4/hhIqg5I3MQFu+pOq/b0XH/
+9I10o6FVU7vcFkQ=
+-----END CERTIFICATE-----

installer/src/test/certs/server.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDMDCCAhgCCQCDr1HQJBr1izANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJV
+UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNDMREwDwYDVQQKDAhDbG91ZHJvbjEe
+MBwGA1UEAwwVaW5zdGFsbGVyLmNsb3Vkcm9uLmlvMB4XDTE1MTExNjIzMTcwMloX
+DTE2MTExNTIzMTcwMlowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQswCQYD
+VQQHDAJTQzERMA8GA1UECgwIQ2xvdWRyb24xHjAcBgNVBAMMFWluc3RhbGxlci5j
+bG91ZHJvbi5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0suQX7
+hKBhYsSH0msnEPVbRDIotYbtVDav/v7Sb/fRU7qVoL31tj2iZRDJRJ27uRM3J4ye
+6hgJAAwQGtfXrcVZY3SOAlGXsFZF0wgBCw0pGtgF3HA1BcwbCwAd06J6w3lKActA
+DMEUio/jRXpYELUU2Nzopq0MsMyyBSBkNC18i0HUB8vkF8yQvb1OpbcxERbpf3D5
+zjeFf5kIE/k8lwBz1vMF0uAA2GfcXxs3dyDaxVteWeevVYZzAoY9EcUyBWX7OQnx
+aUygl3OywN+xOJKXKCQpckzDvr9Vp1sKItoMMy5y81SyNhZIMBYGGG+oNp/wSgQf
+Cht+LupI+bXoYrMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAgPHZx52qYuEUdzVO
+t/+VXO7dxJkONYU8sjTYIfJme8ZZd7beZBMUni5s2gvv6i5HFyJ2Ol88sv8hAaI/
+6Vmbszml+5tLyPK8Gygk62l6OcKDwU/yazTxxCApulNy1SV34kzruXUMZ28ybcqA
+XJywMMx4RDmSIBXPdDCeaOgYwI7Wk56obJ8sa2+Z6100GNoX+qBSOsWMMJW+ohnp
+eQWHkTOJzU4hIMfZCbW0cF5Xn/35xEh0xxaH7XWglJLM9neBPba+Ydz7567mN9co
+vgv2dE5ZOKSjG63CtUvv819dvbWVKq8jiMCqPGRcr1iSeqbC02tnx0W762980uSx
+QfOgAw==
+-----END CERTIFICATE-----

installer/src/test/certs/server.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEArSy5BfuEoGFixIfSaycQ9VtEMii1hu1UNq/+/tJv99FTupWg
+vfW2PaJlEMlEnbu5EzcnjJ7qGAkADBAa19etxVljdI4CUZewVkXTCAELDSka2AXc
+cDUFzBsLAB3TonrDeUoBy0AMwRSKj+NFelgQtRTY3OimrQywzLIFIGQ0LXyLQdQH
+y+QXzJC9vU6ltzERFul/cPnON4V/mQgT+TyXAHPW8wXS4ADYZ9xfGzd3INrFW15Z
+569VhnMChj0RxTIFZfs5CfFpTKCXc7LA37E4kpcoJClyTMO+v1WnWwoi2gwzLnLz
+VLI2FkgwFgYYb6g2n/BKBB8KG34u6kj5tehiswIDAQABAoIBAGNAQ5bbLYsh5ZKP
+6ZhCHqUQtsgsrsVzFhX1zqbLgyK8VUmV4jedMOKoRVZWlD32zj7mGIOuvKoj1mQT
+gt78HPsDnU266jdLQeRgRm/K8UOMsHbo/QtOSFFPmoFpltcDly7XrKmJvwWWOUf4
+UOSqvoCaPyR1Lrn1kQrwaKHE7Ga4jfyOrIq9JI7y/ih+Y7D8xcMnyLAsjyVkSAtr
++XrGNHcx3yPuBmjaOglzeb6Ksdpt4ETElrvH3ByT5EV2zUVr9Txv+m8xSVBZfea9
+aE7lWSQoOUz+e6RhIX3Df/QfR6KkDblAwEF9Se98DWcz46Y34oc2E0lSoJYpoPxP
+vbRlfDkCgYEA3nAc8kDRkbQObSfnVjpijBSP5hfr3jX+XTbxK7Y3aTMViY+87iWK
+bLNuX+2JRCmRjk0wy2YXnJQV3sU/EO5gLhOz9060MIHgFISq4KRgPorN/EFWryOe
+mDzhPIuhZLMetv0ajS3Z5IxIAs+FLu7Yx9em80q540UA3kXsFWe2lpUCgYEAx03E
+kk5zLirVFtoyP/yAES+KVppqBweCUA5vVxB8H26oIhi8G8kT4b77x6wXxQzdsA4H
+a4ou3ZBZVK41PREgG1MWgzpbwk49T1FX6TLtvdhr/9QhYC+RIynynA/pA36LSKT5
+pvWegYB4+9jaPrQ5L1zcrLF2XlTsgpuC43kXKicCgYA0dXxeJatHEY/VbnPAgkR7
+hN3rBfk6jsFOeoamKHMo/EM4Dg4gm/npaOe+9+ZHjQYm6U14qrsm0kXWI+6br5w/
+QaZPzN/yEK8oJ6GlGR8ZoOKzezVWWLAudy0neka12QiFX2vDn+yjWfIht49RYkL9
+3n4hIp50WvG5egQTiEIngQKBgCn9yJzKypm/jIX0EwJIQPNeANeeURiKDHqxj+PY
+JU66EdKdQ4TXKMk3Y/T93UQ3Ib4mNooB4z3rW+brjWwAX7NiHiwn741QzroXeV44
+zL5jCt4r45xQaVPvUp5u+7kwwEfd+nui5HKEjvkBB3qOnj3MYvI/saDOY8Zg3YLv
+0GGhAoGANBwFcDgwP9KDt0NxKXhe3rlSUyfGSSUF89hZPrLDCiaGFURD/w4j3EGr
+Ui9Rcwm2ymqlFzTO4JYKy1/pRCWA7GDfslICJPOPG3Wytsjog0WymQuMjYC2tL/+
+RwD0qG0/aBGE4PbigPRoJ/7BGZLKtdy99P0wyFC3o6OBoAl3Zqo=
+-----END RSA PRIVATE KEY-----

installer/src/test/installer-test.js

@@ -0,0 +1,219 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var expect = require('expect.js'),
+    fs = require('fs'),
+    path = require('path'),
+    nock = require('nock'),
+    os = require('os'),
+    request = require('superagent'),
+    server = require('../server.js'),
+    installer = require('../installer.js'),
+    _ = require('lodash');
+
+var EXTERNAL_SERVER_URL = 'https://localhost:4443';
+var INTERNAL_SERVER_URL = 'http://localhost:2020';
+var APPSERVER_ORIGIN = 'http://appserver';
+var FQDN = os.hostname();
+
+describe('Server', function () {
+    this.timeout(5000);
+
+    before(function (done) {
+        var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string
+        var scope = nock('http://169.254.169.254')
+            .persist()
+            .get('/metadata/v1.json')
+            .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' });
+        done();
+    });
+
+    after(function (done) {
+        nock.cleanAll();
+        done();
+    });
+
+    describe('starts and stop', function () {
+        it('starts', function (done) {
+            server.start(done);
+        });
+
+        it('stops', function (done) {
+            server.stop(done);
+        });
+    });
+
+    describe('update (internal server)', function () {
+        before(function (done) {
+            server.start(done);
+        });
+        after(function (done) {
+            server.stop(done);
+        });
+
+        it('does not respond to provision', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        it('does not respond to restore', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        var data = {
+            sourceTarballUrl: "https://foo.tar.gz",
+
+            data: {
+                token: 'sometoken',
+                apiServerOrigin: APPSERVER_ORIGIN,
+                webServerOrigin: 'https://somethingelse.com',
+                fqdn: 'www.something.com',
+                tlsKey: 'key',
+                tlsCert: 'cert',
+                boxVersionsUrl: 'https://versions.json',
+                version: '0.1'
+            }
+        };
+
+        Object.keys(data).forEach(function (key) {
+            it('fails due to missing ' + key, function (done) {
+                var dataCopy = _.merge({ }, data);
+                delete dataCopy[key];
+
+                request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+            });
+        });
+
+        it('succeeds', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+    });
+
+    describe('retire', function () {
+        var data = {
+            data: {
+                tlsKey: 'key',
+                tlsCert: 'cert'
+            }
+        };
+
+        before(function (done) {
+            process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead
+            server.start(done);
+        });
+
+        after(function (done) {
+            server.stop(done);
+            delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+        });
+
+        Object.keys(data).forEach(function (key) {
+            it('fails due to missing ' + key, function (done) {
+                var dataCopy = _.merge({ }, data);
+                delete dataCopy[key];
+
+                request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(dataCopy).end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+            });
+        });
+
+        it('succeeds', function (done) {
+            request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(data).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+    });
+
+    describe('ensureVersion', function () {
+        before(function () {
+            process.env.NODE_ENV = undefined;
+        });
+
+        after(function () {
+            process.env.NODE_ENV = 'test';
+        });
+
+        it ('fails without data', function (done) {
+            installer._ensureVersion({}, function (error) {
+                expect(error).to.be.an(Error);
+                done();
+            });
+        });
+
+        it ('fails without boxVersionsUrl', function (done) {
+            installer._ensureVersion({ data: {}}, function (error) {
+                expect(error).to.be.an(Error);
+                done();
+            });
+        });
+
+        it ('succeeds with sourceTarballUrl', function (done) {
+            var data = {
+                sourceTarballUrl: 'sometarballurl',
+                data: {
+                    boxVersionsUrl: 'http://foobar/versions.json'
+                }
+            };
+
+            installer._ensureVersion(data, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result).to.eql(data);
+                done();
+            });
+        });
+
+        it ('succeeds without sourceTarballUrl', function (done) {
+            var versions = {
+                '0.1.0': {
+                    sourceTarballUrl: 'sometarballurl1'
+                },
+                '0.2.0': {
+                    sourceTarballUrl: 'sometarballurl2'
+                }
+            };
+
+            var scope = nock('http://foobar')
+                .get('/versions.json')
+                .reply(200, JSON.stringify(versions), { 'Content-Type': 'application/json' });
+
+            var data = {
+                data: {
+                    boxVersionsUrl: 'http://foobar/versions.json'
+                }
+            };
+
+            installer._ensureVersion(data, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result.sourceTarballUrl).to.equal(versions['0.2.0'].sourceTarballUrl);
+                expect(result.data.boxVersionsUrl).to.equal(data.data.boxVersionsUrl);
+                done();
+            });
+        });
+    });
+});
+

installer/systemd/box-setup.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly USER_HOME="/home/yellowtent"
+readonly APPS_SWAP_FILE="/apps.swap"
+readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups
+readonly USER_DATA_FILE="/root/user_data.img"
+readonly USER_DATA_DIR="/home/yellowtent/data"
+
+# detect device
+if [[ -b "/dev/vda1" ]]; then
+    disk_device="/dev/vda1"
+fi
+
+if [[ -b "/dev/xvda1" ]]; then
+    disk_device="/dev/xvda1"
+fi
+
+# all sizes are in mb
+readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
+readonly swap_size="${physical_memory}"
+readonly app_count=$((${physical_memory} / 200)) # estimated app count
+readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
+readonly disk_size=$((disk_size_gb * 1024))
+readonly backup_swap_size=1024
+# readonly system_size=5120 # 5 gigs for system libs, installer, box code and tmp
+readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
+readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
+
+echo "Disk device: ${disk_device}"
+echo "Physical memory: ${physical_memory}"
+echo "Estimated app count: ${app_count}"
+echo "Disk size: ${disk_size}"
+
+# Allocate two sets of swap files - one for general app usage and another for backup
+# The backup swap is setup for swap on the fly by the backup scripts
+if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
+    echo "Creating Apps swap file of size ${swap_size}M"
+    fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
+    chmod 600 "${APPS_SWAP_FILE}"
+    mkswap "${APPS_SWAP_FILE}"
+    swapon "${APPS_SWAP_FILE}"
+    echo "${APPS_SWAP_FILE}  none  swap  sw  0 0" >> /etc/fstab
+else
+    echo "Apps Swap file already exists"
+fi
+
+if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then
+    echo "Creating Backup swap file of size ${backup_swap_size}M"
+    fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}"
+    chmod 600 "${BACKUP_SWAP_FILE}"
+    mkswap "${BACKUP_SWAP_FILE}"
+else
+    echo "Backups Swap file already exists"
+fi
+
+echo "Resizing data volume"
+home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
+echo "Resizing up btrfs user data to size ${home_data_size}M"
+umount "${USER_DATA_DIR}"
+fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
+mount "${USER_DATA_FILE}"
+btrfs filesystem resize max "${USER_DATA_DIR}"
+

migrations/20141021192552-db-create.js

@@ -1,8 +1,12 @@
 var dbm = require('db-migrate');
 var type = dbm.dataType;
+var url = require('url');
 
 exports.up = function(db, callback) {
-    callback();
+    var dbName = url.parse(process.env.DATABASE_URL).path.substr(1); // remove slash
+
+    // by default, mysql collates case insensitively. 'utf8_general_cs' is not available
+    db.runSql('ALTER DATABASE ' + dbName + '  DEFAULT CHARACTER SET=utf8 DEFAULT COLLATE utf8_bin', callback);
 };
 
 exports.down = function(db, callback) {

migrations/20141021235251-clients-add-webadmin.js

@@ -1,21 +0,0 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
-
-var uuid = require('node-uuid');
-
-exports.up = function(db, callback) {
-    var scopes = 'root,profile,users,apps,settings,roleAdmin';
-    var adminOrigin = 'https://admin-localhost';
-
-    // postinstall.sh creates the webadmin entry in production mode
-    if (process.env.NODE_ENV !== 'test') return callback(null);
-
-    db.runSql('INSERT INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) ' +
-              'VALUES (?, ?, ?, ?, ?, ?, ?)', [ uuid.v4(), 'webadmin', 'cid-webadmin', 'unused', 'WebAdmin', adminOrigin, scopes ],
-              callback);
-};
-
-exports.down = function(db, callback) {
-    // not sure what is meaningful here
-    callback(null);
-};

migrations/20141022153739-settings-add-naked-domain.js

@@ -1,10 +0,0 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('INSERT INTO settings (key, value) VALUES (?, ?)', [ 'naked_domain', null ], callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DELETE FROM settings WHERE key=?', [ 'naked_domain' ], callback);
-};

migrations/20141128175100-add-addonconfigs.js

@@ -1,15 +0,0 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
-
-exports.up = function(db, callback) {
-    db.runSql('CREATE TABLE appAddonConfigs(' +
-              ' appId VARCHAR(512) NOT NULL,' +
-              ' addonId VARCHAR(32) NOT NULL,' +
-              ' value VARCHAR(512) NOT NULL,' +
-              ' FOREIGN KEY(appId) REFERENCES apps(id))', callback);
-};
-
-exports.down = function(db, callback) {
-    db.runSql('DROP TABLE appAddonConfigs', callback);
-};
-

migrations/20150303114527-users-add-resetToken.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN resetToken VARCHAR(128) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN resetToken', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150303160528-tokens-expires-bigint.js

@@ -0,0 +1,20 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM tokens', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE tokens MODIFY expires BIGINT', [], function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens MODIFY expires VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150303171203-authcodes-add-expiresAt.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE authcodes ADD COLUMN expiresAt BIGINT NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE authcodes DROP COLUMN expiresAt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150308054950-appportbindings-add-environmentVariable.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings ADD COLUMN environmentVariable VARCHAR(128) NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings DROP COLUMN environmentVariable', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150309023251-appportbindings-drop-containerport.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings DROP COLUMN containerPort', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appPortBindings ADD COLUMN containerPort VARCHAR(5) NOT NULL', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150311120713-tokens-rename-userid-to-identifier.js

@@ -0,0 +1,20 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM tokens', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE tokens CHANGE userId identifier VARCHAR(128) NOT NULL', [], function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE tokens CHANGE identifier userId VARCHAR(128) NOT NULL', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150323044254-apps-drop-version.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN version', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN version VARCHAR(32)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150323174906-apps-alter-health.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN healthy, ADD COLUMN health VARCHAR(128)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN health, ADD COLUMN healthy INTEGER', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20150326072904-apps-add-lastBackupId.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupId VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupId', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150404093025-apps-add-createdAt.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN createdAt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150430165335-settings-default-autoupdatepattern.js

@@ -0,0 +1,12 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    // everyday at 1am
+    db.runSql('INSERT settings (name, value) VALUES("autoupdate_pattern", ?)', [ '00 00 1 * * *' ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="autoupdate_pattern"', [ ], callback);
+}
+

migrations/20150430210225-settings-default-timezone.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+var safe = require('safetydance');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
+    tz = tz ? tz.trim() : 'America/Los_Angeles';
+
+    db.runSql('INSERT settings (name, value) VALUES("time_zone", ?)', [ tz ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="time_zone"', [ ], callback);
+};
+

migrations/20150615075901-users-add-unique-constraints.js

@@ -0,0 +1,24 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+
+    // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address
+
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_username UNIQUE (username)'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users ADD CONSTRAINT users_email UNIQUE (email)'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_username'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(512)'),
+        db.runSql.bind(db, 'ALTER TABLE users DROP INDEX users_email'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(512)'),
+    ], callback);
+};

migrations/20150615134751-users-adjust-username-and-email-constraints.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254) NOT NULL'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY username VARCHAR(254)'),
+        db.runSql.bind(db, 'ALTER TABLE users MODIFY email VARCHAR(254)'),
+    ], callback);
+};

migrations/20150618192028-apps-add-lastManifestJson.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastManifestJson VARCHAR(2048)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastManifestJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150710044740-apps-rename-lastBackupConfigJson.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastManifestJson lastBackupConfigJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastBackupConfigJson lastManifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150710170847-apps-add-oldConfigJson.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oldConfigJson VARCHAR(2048)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oldConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20150719014338-settings-remove-all.js

@@ -0,0 +1,10 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('DELETE FROM settings', [ ], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20151013073519-apps-add-oauthProxy.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151015232915-clients-add-type.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'DELETE FROM clients'),
+        db.runSql.bind(db, 'ALTER TABLE clients ADD COLUMN type VARCHAR(16) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE clients DROP COLUMN type', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151016131005-apps-rename-accessRestriction.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestriction accessRestrictionJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestrictionJson accessRestriction VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151113091257-apps-alter-manifestJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151126111337-apps-alter-jsons.js

@@ -0,0 +1,19 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson TEXT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson VARCHAR(2048)')
+    ], callback);
+};

migrations/initial-schema.sql

@@ -2,64 +2,66 @@ CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
     username VARCHAR(512) NOT NULL,
     email VARCHAR(512) NOT NULL,
-    _password VARCHAR(512) NOT NULL,
-    publicPem VARCHAR(2048) NOT NULL,
-    _privatePemCipher VARCHAR(2048) NOT NULL,
-    _salt VARCHAR(512) NOT NULL,
+    password VARCHAR(1024) NOT NULL,
+    salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS tokens(
-    accessToken VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512),
+    accessToken VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
     expires VARCHAR(512) NOT NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appId VARCHAR(128) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
-    name VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS apps(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appStoreId VARCHAR(512) NOT NULL,
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appStoreId VARCHAR(128) NOT NULL,
     version VARCHAR(32),
     installationState VARCHAR(512) NOT NULL,
     installationProgress VARCHAR(512),
     runState VARCHAR(512),
     healthy INTEGER,
     containerId VARCHAR(128),
-    manifestJson VARCHAR,
+    manifestJson VARCHAR(2048),
     httpPort INTEGER,
-    location VARCHAR(512) NOT NULL UNIQUE,
+    location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
     accessRestriction VARCHAR(512),
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
-    hostPort VARCHAR(5) NOT NULL UNIQUE,
+    hostPort INTEGER NOT NULL UNIQUE,
     containerPort VARCHAR(5) NOT NULL,
-    appId VARCHAR(512) NOT NULL,
+    appId VARCHAR(128) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
 CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    authCode VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128) NOT NULL,
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
-    key VARCHAR(512) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL UNIQUE,
     value VARCHAR(512),
-    PRIMARY KEY(key));
+    PRIMARY KEY(name));
+
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(128) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id));
 

migrations/schema.sql

@@ -1,74 +1,87 @@
 #### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 #### This file is not used by any code and is here to document the latest schema
 
+#### General ideas
+#### Default char set is utf8 and DEFAULT COLLATE is utf8_bin. Collate affects comparisons in WHERE and ORDER
+#### Strict mode is enabled
+#### VARCHAR - stored as part of table row (use for strings)
+#### TEXT - stored offline from table row (use for strings)
+#### BLOB - stored offline from table row (use for binary data)
+#### https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
+
 CREATE TABLE IF NOT EXISTS users(
     id VARCHAR(128) NOT NULL UNIQUE,
-    username VARCHAR(512) NOT NULL,
-    email VARCHAR(512) NOT NULL,
-    _password VARCHAR(512) NOT NULL,
-    publicPem VARCHAR(2048) NOT NULL,
-    _privatePemCipher VARCHAR(2048) NOT NULL,
-    _salt VARCHAR(512) NOT NULL,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    email VARCHAR(254) NOT NULL UNIQUE,
+    password VARCHAR(1024) NOT NULL,
+    salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS tokens(
-    accessToken VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512),
+    accessToken VARCHAR(128) NOT NULL UNIQUE,
+    identifier VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
-    expires VARCHAR(512) NOT NULL,
+    expires BIGINT NOT NULL,
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
+    appId VARCHAR(128) NOT NULL,
+    type VARCHAR(16) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
-    name VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS apps(
-    id VARCHAR(512) NOT NULL UNIQUE,
-    appStoreId VARCHAR(512) NOT NULL,
-    version VARCHAR(32),
+    id VARCHAR(128) NOT NULL UNIQUE,
+    appStoreId VARCHAR(128) NOT NULL,
     installationState VARCHAR(512) NOT NULL,
     installationProgress VARCHAR(512),
     runState VARCHAR(512),
-    healthy INTEGER,
+    health VARCHAR(128),
     containerId VARCHAR(128),
-    manifestJson VARCHAR,
-    httpPort INTEGER,
-    location VARCHAR(512) NOT NULL UNIQUE,
+    manifestJson TEXT,
+    httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
+    location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestriction VARCHAR(512),
+    accessRestrictionJson TEXT,
+    oauthProxy BOOLEAN DEFAULT 0,
+    createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    lastBackupId VARCHAR(128),
+    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+
+    oldConfigJson TEXT, // used to pass old config for apptask
+
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
-    hostPort VARCHAR(5) NOT NULL UNIQUE,
-    containerPort VARCHAR(5) NOT NULL,
-    appId VARCHAR(512) NOT NULL,
+    hostPort INTEGER NOT NULL UNIQUE,
+    environmentVariable VARCHAR(128) NOT NULL,
+    appId VARCHAR(128) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id),
     PRIMARY KEY(hostPort));
 
-CREATE TABLE IF NOT EXISTS appAddonConfigs(
-    appId VARCHAR(512) NOT NULL,
-    addonId VARCHAR(32) NOT NULL,
-    value VARCHAR(512),
-    FOREIGN KEY(appId) REFERENCES apps(id));
-
 CREATE TABLE IF NOT EXISTS authcodes(
-    authCode VARCHAR(512) NOT NULL UNIQUE,
-    userId VARCHAR(512) NOT NULL,
-    clientId VARCHAR(512) NOT NULL,
+    authCode VARCHAR(128) NOT NULL UNIQUE,
+    userId VARCHAR(128) NOT NULL,
+    clientId VARCHAR(128) NOT NULL,
+    expiresAt BIGINT NOT NULL,
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
-    key VARCHAR(512) NOT NULL UNIQUE,
+    name VARCHAR(128) NOT NULL UNIQUE,
     value VARCHAR(512),
-    PRIMARY KEY(key));
+    PRIMARY KEY(name));
+
+CREATE TABLE IF NOT EXISTS appAddonConfigs(
+    appId VARCHAR(128) NOT NULL,
+    addonId VARCHAR(32) NOT NULL,
+    value VARCHAR(512) NOT NULL,
+    FOREIGN KEY(appId) REFERENCES apps(id));
 

oauthproxy.js

@@ -1,119 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var express = require('express'),
-    url = require('url'),
-    async = require('async'),
-    assert = require('assert'),
-    debug = require('debug')('box:proxy'),
-    proxy = require('proxy-middleware'),
-    session = require('cookie-session'),
-    database = require('./src/database.js'),
-    appdb = require('./src/appdb.js'),
-    clientdb = require('./src/clientdb.js'),
-    config = require('./config.js'),
-    http = require('http');
-
-var gSessions = {};
-var gProxyMiddlewareCache = {};
-var gApp = express();
-var gHttpServer = http.createServer(gApp);
-
-var CALLBACK_URI = '/callback';
-var PORT = 4000;
-
-function startServer(callback) {
-    assert(typeof callback === 'function');
-
-    gHttpServer.on('error', console.error);
-
-    gApp.use(session({
-        keys: ['blue', 'cheese', 'is', 'something']
-    }));
-
-    gApp.use(function (req, res, next) {
-        if (req.session && gSessions[req.session.sessid]) return next();
-
-        if (req.path === CALLBACK_URI) {
-            // FIXME we need to exchange the authCode and verify it
-            req.session.sessid = req.query.authCode;
-
-            // this is a simple in memory auth store
-            gSessions[req.session.sessid] = 'ok';
-
-            debug('user verified.');
-
-            // now redirect to the actual initially requested URL
-            res.redirect(req.session.returnTo);
-        } else {
-            var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
-
-            if (!Number.isFinite(port)) {
-                console.error('Failed to parse nginx proxy header to get app port.');
-                return res.send(500, 'Routing error. No forwarded port.');
-            }
-
-            debug('begin verifying user for app on port %s.', port);
-
-            appdb.getByHttpPort(port, function (error, result) {
-                if (error) {
-                    console.error('Unknown app.', error);
-                    return res.send(500, 'Unknown app.');
-                }
-
-                clientdb.getByAppId('proxy-' + result.id, function (error, result) {
-                    if (error) {
-                        console.error('Unkonwn OAuth client.', error);
-                        return res.send(500, 'Unknown OAuth client.');
-                    }
-
-                    req.session.port = port;
-                    req.session.returnTo =  result.redirectURI + req.path;
-
-                    var callbackURL = result.redirectURI + CALLBACK_URI;
-                    var scope = 'profile,roleUser';
-                    var clientId = result.clientId;
-                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + clientId + '&redirect_uri=' + callbackURL + '&scope=' + scope;
-
-                    debug('begin OAuth flow for client %s.', result.name);
-
-                    // begin the OAuth flow
-                    res.redirect(oauthLogin);
-                });
-            });
-        }
-    });
-
-    gApp.use(function (req, res, next) {
-        var port = req.session.port;
-
-        debug('proxy request for port %s with path %s.', port, req.path);
-
-        var proxyMiddleware = gProxyMiddlewareCache[port];
-        if (!proxyMiddleware) {
-            console.log('Adding proxy middleware for port %d', port);
-
-            proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
-            gProxyMiddlewareCache[port] = proxyMiddleware;
-        }
-
-        proxyMiddleware(req, res, next);
-    });
-
-    gHttpServer.listen(PORT, callback);
-}
-
-async.series([
-    database.initialize,
-    startServer
-], function (error) {
-    if (error) {
-        console.error('Failed to start proxy server.', error);
-        process.exit(1);
-    }
-
-    console.log('Proxy server listening...');
-});

package.json

@@ -1,96 +1,101 @@
 {
-  "name": "yellowtent",
-  "description": "Yellow tent",
+  "name": "Cloudron",
+  "description": "Main code for a cloudron",
   "version": "0.0.1",
   "private": "true",
   "author": {
-    "name": "Yellow tent authors",
-    "email": "girish@forwardbias.in"
+    "name": "Cloudron authors"
   },
   "repository": {
     "type": "git"
   },
   "engines": [
-    "node >= 0.10.0"
+    "node >=4.0.0 <=4.1.1"
   ],
-  "bin": {
-    "yellowtent": "./server.js"
-  },
   "dependencies": {
-    "async": "^0.6.2",
-    "body-parser": "~1.9.3",
-    "commander": "^2.2.0",
+    "async": "^1.2.1",
+    "attempt": "^1.0.1",
+    "aws-sdk": "^2.1.46",
+    "body-parser": "^1.13.1",
+    "bytes": "^2.1.0",
+    "cloudron-manifestformat": "^2.2.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.8",
-    "connect-timeout": "~1.4.0",
-    "cookie-parser": "1.1.0",
+    "connect-lastmile": "0.0.13",
+    "connect-timeout": "^1.5.0",
+    "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
-    "csurf": "^1.6.1",
-    "db-migrate": "~0.7.1",
-    "debug": "~0.8.1",
-    "dockerode": "~2.0.5",
-    "ejs": "^1.0.0",
-    "encfs": "^0.1.1",
-    "express": "~4.2.0",
-    "express-session": "~1.1.0",
-    "js-yaml": "~3.2.2",
-    "json": "~9.0.1",
-    "memorystream": "~0.2.0",
-    "mime": "^1.2.11",
-    "mkdirp": "~0.3.5",
-    "morgan": "~1.0.1",
-    "multiparty": "http://registry.npmjs.org/multiparty/-/multiparty-4.0.0.tgz",
-    "native-dns": "~0.6.1",
-    "node-uuid": "^1.4.1",
-    "nodejs-disks": "~0.2.1",
-    "nodemailer": "~1.3.0",
-    "nodemailer-smtp-transport": "~0.1.13",
+    "cron": "^1.0.9",
+    "csurf": "^1.6.6",
+    "db-migrate": "^0.9.2",
+    "debug": "^2.2.0",
+    "dockerode": "^2.2.2",
+    "ejs": "^2.2.4",
+    "ejs-cli": "^1.0.1",
+    "express": "^4.12.4",
+    "express-session": "^1.11.3",
+    "hat": "0.0.3",
+    "json": "^9.0.3",
+    "ldapjs": "^0.7.1",
+    "memorystream": "^0.3.0",
+    "mime": "^1.3.4",
+    "morgan": "^1.6.0",
+    "multiparty": "^4.1.2",
+    "mysql": "^2.7.0",
+    "native-dns": "^0.7.0",
+    "node-uuid": "^1.4.3",
+    "nodemailer": "^1.3.0",
+    "nodemailer-smtp-transport": "^1.0.3",
     "oauth2orize": "^1.0.1",
-    "once": "^1.3.0",
-    "passport": "~0.2.1",
+    "once": "^1.3.2",
+    "passport": "^0.2.2",
     "passport-http": "^0.2.2",
     "passport-http-bearer": "^1.0.1",
     "passport-local": "^1.0.0",
-    "passport-oauth2-client-password": "~0.1.2",
-    "password-generator": "~0.2.3",
-    "proxy-middleware": "~0.5.1",
-    "readdirp": "^1.0.1",
-    "rimraf": "^2.2.6",
-    "safetydance": "0.0.12",
-    "semver": "~4.2.0",
-    "serve-favicon": "~2.1.7",
-    "split": "^0.3.0",
-    "sqlite3": "^3.0.0",
-    "superagent": "~0.17.0",
-    "supererror": "~0.6.0",
-    "underscore": "~1.7.0",
-    "ursa": "^0.8.0",
-    "validator": "~3.22.1"
+    "passport-oauth2-client-password": "^0.1.2",
+    "password-generator": "^2.0.2",
+    "proxy-middleware": "^0.13.0",
+    "safetydance": "^0.1.0",
+    "semver": "^4.3.6",
+    "serve-favicon": "^2.2.0",
+    "split": "^1.0.0",
+    "superagent": "^1.5.0",
+    "supererror": "^0.7.1",
+    "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "underscore": "^1.7.0",
+    "ursa": "^0.9.1",
+    "valid-url": "^1.0.9",
+    "validator": "^4.4.0",
+    "x509": "^0.2.2"
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "~2.0.23",
+    "bootstrap-sass": "^3.3.3",
     "del": "^1.1.1",
     "expect.js": "*",
-    "gulp": "^3.8.10",
+    "gulp": "^3.8.11",
+    "gulp-autoprefixer": "^2.3.0",
     "gulp-concat": "^2.4.3",
     "gulp-ejs": "^1.0.0",
-    "gulp-sourcemaps": "^1.3.0",
-    "hock": "~0.2.5",
-    "husky": "~0.6.2",
+    "gulp-minify-css": "^1.1.3",
+    "gulp-sass": "^2.0.1",
+    "gulp-serve": "^1.0.0",
+    "gulp-sourcemaps": "^1.5.2",
+    "gulp-uglify": "^1.1.0",
+    "hock": "~1.2.0",
     "istanbul": "*",
+    "js2xmlparser": "^1.0.0",
     "mocha": "*",
-    "nock": "~0.43.1",
-    "redis": "~0.12.1",
-    "s3-cli": "~0.11.1",
-    "semver": "~4.2.0",
-    "sinon": "~1.10.3"
+    "nock": "^3.4.0",
+    "node-sass": "^3.0.0-alpha.0",
+    "redis": "^2.4.2",
+    "request": "^2.65.0",
+    "sinon": "^1.12.2",
+    "yargs": "^3.15.0"
   },
   "scripts": {
-    "create_testdb": "rm -rf $HOME/.yellowtenttest/*; mkdir -p $HOME/.yellowtenttest/data; NODE_ENV=test DATABASE_URL=sqlite3:///$HOME/.yellowtenttest/data/cloudron.sqlite node_modules/.bin/db-migrate up",
-    "migrate": "mkdir -p $HOME/.yellowtent/data; DATABASE_URL=sqlite3:///$HOME/.yellowtent/data/cloudron.sqlite node_modules/.bin/db-migrate up",
-    "migrate_data": "DATABASE_URL=sqlite3:///home/yellowtent/data/cloudron.sqlite db-migrate up",
-    "test": "scripts/checkInstall && npm run-script create_testdb && NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
+    "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
+    "migrate_test": "BOX_ENV=test DATABASE_URL=mysql://root:@localhost/boxtest node_modules/.bin/db-migrate up",
+    "test": "npm run migrate_test && src/test/setupTest && BOX_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test ./src/routes/test",
     "postmerge": "/bin/true",
     "precommit": "/bin/true",
     "prepush": "npm test",

release/publish.sh

@@ -1,144 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-readonly JSON="${SOURCE_DIR}/node_modules/.bin/json"
-[ "$(uname -s)" == "Darwin" ] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-readonly VERSIONS_URL_DEV="https://s3.amazonaws.com/cloudron-releases/versions-dev.json"
-readonly VERSIONS_S3_URL_DEV="s3://cloudron-releases/versions-dev.json"
-readonly VERSIONS_URL_STAGING="https://s3.amazonaws.com/cloudron-releases/versions-staging.json"
-readonly VERSIONS_S3_URL_STAGING="s3://cloudron-releases/versions-staging.json"
-
-if [[ ! -f "${SOURCE_DIR}/../installer/scripts/digitalOceanFunctions.sh" ]]; then
-    echo "Could not locate digitalOceanFunctions.sh"
-    exit 1
-fi
-
-source "${SOURCE_DIR}/../installer/scripts/digitalOceanFunctions.sh"
-
-new_versions_file=""
-source_tarball_url=""
-image_id=""
-cmd=""
-new_version=""
-changelog="If I told you, I'd have to kill you"
-upgrade="autodetect"
-versions_url="${VERSIONS_URL_DEV}"
-versions_s3_url="${VERSIONS_S3_URL_DEV}"
-
-args=$($GNU_GETOPT -o "" -l "dev,staging,code:,image:,rerelease,new,list,revert,changelog:,release:,upgrade" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --dev) shift;;
-    --staging) versions_url="${VERSIONS_URL_STAGING}"; versions_s3_url="${VERSIONS_S3_URL_STAGING}"; shift;;
-    --code) source_tarball_url="$2"; shift 2;;
-    --image) image_id="$2"; shift 2;;
-    --rerelease) cmd="rerelease"; shift;;
-    --new) cmd="new"; shift;;
-    --release) cmd="release"; new_versions_file="$2"; shift 2;;
-    --list) cmd="list"; shift;;
-    --revert) cmd="revert"; shift;;
-    --changelog) changelog="$2"; shift 2;;
-    --upgrade) upgrade="true"; shift;;
-    --) shift; break;;
-    *) echo "Unknown option $2"; exit;;
-    esac
-done
-
-shift $(expr $OPTIND - 1)
-
-download_current() {
-    versions_url="$1"
-    # download the existing version file if the user hasn't provided one
-    local current_versions_file=$(mktemp -t box-versions 2>/dev/null || mktemp)
-
-    if ! wget -q -O "${current_versions_file}" "${versions_url}"; then
-        echo "Error downloading versions file"
-        exit 1
-    fi
-
-    echo "${current_versions_file}"
-}
-
-if [[ "${cmd}" == "list" ]]; then
-    cat "$(download_current "${versions_url}")"
-    exit 0
-elif [[ "${cmd}" == "release" ]]; then
-    if [[ ! -f "${new_versions_file}" ]]; then
-        echo "${new_versions_file} cannot be found"
-        exit 1
-    fi
-elif [[ "${cmd}" == "new" ]]; then
-    if [[ -z "${source_tarball_url}" || -z "${image_id}" ]]; then
-        echo "--code and --image is required"
-        exit 1
-    fi
-
-    new_version="0.0.1"
-    image_name=$(get_image_name "${image_id}")
-
-    new_versions_file=$(mktemp -t box-versions 2>/dev/null || mktemp)
-    cat > "${new_versions_file}" <<EOF
-    {
-        "0.0.1": {
-            "sourceTarballUrl": "${source_tarball_url}",
-            "imageId": ${image_id},
-            "imageName": "${image_name}",
-            "changelog": [ "Let's start at the very beginning, a very good way to start" ],
-            "date": "$(date -u)",
-            "next": null
-        }
-    }
-EOF
-elif [[ "${cmd}" == "revert" ]]; then
-    new_versions_file=$(download_current "${versions_url}")
-    last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 1)
-    second_last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 2 | head -n 1)
-
-    echo "Removing $last_version and making $second_last_version the last release"
-    $JSON -q -I -f "${new_versions_file}" -e "delete this['${last_version}']"
-    $JSON -q -I -f "${new_versions_file}" -e "this['${second_last_version}'].next = null"
-else
-    new_versions_file=$(download_current "${versions_url}")
-    # modify existing versions.json
-    if [[ -z "${source_tarball_url}" && -z "${image_id}" && "${cmd}" != "rerelease" ]]; then
-        echo "--code or --image is required"
-        exit 1
-    fi
-
-    readonly last_version=$(cat "${new_versions_file}" | $JSON -ka | tail -n 1)
-    if [[ -z "${source_tarball_url}" ]]; then
-        source_tarball_url=$($JSON -f "${new_versions_file}" -D, "${last_version},sourceTarballUrl")
-        echo "Using the previous code url : ${source_tarball_url}"
-    fi
-    if [[ -z "${image_id}" ]]; then
-        image_id=$($JSON -f "${new_versions_file}" -D, "${last_version},imageId")
-        echo "Using the previous image id : ${image_id}"
-    fi
-
-    if [[ "${upgrade}" == "autodetect" ]]; then
-        old_image_id=$($JSON -f "${new_versions_file}" -D, "${last_version},imageId")
-        upgrade=$([[ "${old_image_id}" != "${image_id}" ]] && echo "true" || echo "false")
-    fi
-
-    new_version=$($SOURCE_DIR/node_modules/.bin/semver -i "${last_version}")
-    echo "Releasing version ${new_version}"
-    image_name=$(get_image_name "${image_id}")
-
-    $JSON -q -I -f "${new_versions_file}" -e "this['${last_version}'].next = '${new_version}'"
-    $JSON -q -I -f "${new_versions_file}" -e "this['${new_version}'] = { 'sourceTarballUrl': '${source_tarball_url}', 'imageId': ${image_id}, 'imageName': '${image_name}', 'changelog': [ \"${changelog}\" ], 'upgrade': ${upgrade}, 'date': '$(date -u)', 'next': null }"
-fi
-
-echo "Verifying new versions file"
-$SOURCE_DIR/release/verify.js "${new_versions_file}"
-
-echo "Uploading new versions file"
-$SOURCE_DIR/node_modules/.bin/s3-cli put --acl-public --default-mime-type "application/json" "${new_versions_file}" "${versions_s3_url}"
-
-cat "${new_versions_file}"
-

release/stage.sh

@@ -1,102 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-readonly JSON="${SOURCE_DIR}/node_modules/.bin/json"
-readonly SEMVER="${SOURCE_DIR}/node_modules/.bin/semver"
-[ $(uname -s) == "Darwin" ] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-readonly VERSIONS_URL_DEV="https://s3.amazonaws.com/cloudron-releases/versions-dev.json"
-readonly VERSIONS_URL_STAGING="https://s3.amazonaws.com/cloudron-releases/versions-staging.json"
-readonly VERSIONS_S3_URL_STAGING="s3://cloudron-releases/versions-staging.json"
-
-verify_tag() {
-    tag="$1"
-    git rev-parse --verify "tags/$1" 2>/dev/null
-}
-
-download() {
-    # download the existing version file if the user hasn't provided one
-    local tmp_file=$(mktemp -t stage 2>/dev/null || mktemp)
-
-    if wget -q -O "${tmp_file}" "${1}"; then
-        echo "${tmp_file}"
-    fi
-}
-
-read_changelog() {
-    version="$1"
-    changelog_file="${SOURCE_DIR}/release/changelogs/changelog-$1"
-
-    if [[ -f "${changelog_file}" ]]; then
-        cat "${changelog_file}" | grep -v "^#"
-    fi
-}
-
-if [[ $# -lt 1 ]]; then
-    echo "Usage: stage.sh <dev-version>"
-    exit 1
-fi
-
-dev_version="$1"
-
-dev_versions_file=$(download "${VERSIONS_URL_DEV}")
-if [[ -z "${dev_versions_file}" ]]; then
-    echo "Error downloading dev versions file"
-    exit 1
-fi
-
-dev_version_info=$($JSON -f "${dev_versions_file}" -D, "${dev_version}")
-if [[ -z "${dev_version_info}" ]]; then
-    echo "No such version in dev ${dev_versions_file} ${dev_version}"
-    exit 1
-fi
-
-staging_versions_file=$(download "${VERSIONS_URL_STAGING}") ## TODO: this can fail
-if [[ -z "${staging_versions_file}" ]]; then
-    echo "Creating new staging release file"
-    staging_versions_file=$(mktemp -t stage 2>/dev/null || mktemp)
-    echo "{}" > ${staging_versions_file}
-    readonly staging_last_version="0.0.0"
-    staging_new_version="0.0.1"
-    upgrade="false"
-else
-    readonly staging_last_version=$(cat "${staging_versions_file}" | $JSON -ka | tail -n 1)
-    staging_new_version=$($SEMVER -i "${staging_last_version}")
-    $JSON -q -I -f "${staging_versions_file}" -e "this['${staging_last_version}'].next = '${staging_new_version}'"
-
-    last_image_id=$($JSON -f "${staging_versions_file}" -D, "${staging_last_version},imageId")
-    new_image_id=$($JSON -f "${dev_versions_file}" -D, "${dev_version},imageId")
-
-    upgrade=$([[ "${last_image_id}" != "${new_image_id}" ]] && echo "true" || echo "false")
-fi
-
-#TODO: check if the tag matches the sha1 in the sourceTarballUrl
-if ! verify_tag "v${staging_new_version}"; then
-    echo "No git tag named v${staging_new_version} found"
-    exit 1
-fi
-
-changelog=$(read_changelog "${staging_new_version}")
-if [[ -z "${changelog}" ]]; then
-    echo "Missing changelog file or empty change log"
-    exit 1
-fi
-
-echo "Releasing version ${staging_new_version}"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'] = ${dev_version_info}"
-#$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].changelog = '[ "${changelog}" ]'"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].upgrade = ${upgrade}"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].date = '$(date -u)'"
-$JSON -q -I -f "${staging_versions_file}" -e "this['${staging_new_version}'].next = null"
-
-echo "Verifying new versions file"
-$SOURCE_DIR/release/verify.js "${staging_versions_file}"
-
-echo "Uploading new versions file"
-$SOURCE_DIR/node_modules/.bin/s3-cli put --acl-public --default-mime-type "application/json" "${staging_versions_file}" "${VERSIONS_S3_URL_STAGING}"
-
-cat "${staging_versions_file}" | tee $SOURCE_DIR/release/versions-staging.json
-

release/verify.js

@@ -1,55 +0,0 @@
-#!/usr/bin/env node
-
-var AWS = require('aws-sdk'),
-    fs = require('fs'),
-    path = require('path'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    url = require('url');
-
-function die(msg) {
-    console.error(msg);
-    process.exit(1);
-}
-
-function verify(versionsFileName) {
-    // check if the json is valid
-    var versionsJson = safe.JSON.parse(fs.readFileSync(versionsFileName));
-    if (!versionsJson) {
-        die(versionsFileName + ' is not valid json : ' + safe.error);
-    }
-
-    // check all the keys
-    var sortedVersions = Object.keys(versionsJson).sort();
-    sortedVersions.forEach(function (version, index) {
-        if (typeof versionsJson[version].imageId !== 'number') die('version ' + version + ' does not have proper imageId');
-        if (typeof versionsJson[version].imageName !== 'string' || !versionsJson[version].imageName.length) die('version ' + version + ' does not have proper imageName');
-        if ('changeLog' in versionsJson[version] && !util.isArray(versionsJson[version].changeLog)) die('version ' + version + ' does not have proper changeLog');
-        if (typeof versionsJson[version].date !== 'string' || ((new Date(versionsJson[version].date)).toString() === 'Invalid Date')) die('invalid date or missing date');
-        if (versionsJson[version].next !== null && typeof versionsJson[version].next !== 'string') die('version ' + version + ' does not have proper next');
-        if (typeof versionsJson[version].sourceTarballUrl !== 'string') die('version ' + version + ' does not have proper sourceTarballUrl');
-        var tarballUrl = url.parse(versionsJson[version].sourceTarballUrl);
-        if (tarballUrl.protocol !== 'https:') die('sourceTarballUrl must be https');
-        if (!/.tar.gz$/.test(tarballUrl.path)) die('sourceTarballUrl must be tar.gz');
-
-        var nextVersion = versionsJson[version].next;
-        // despite having the 'next' field, the appstore code currently relies on all versions being sorted based on semver.compare (see boxversions.js)
-        if (nextVersion && semver.gt(version, nextVersion)) die('next version cannot be less than current @' + version);
-    });
-
-    // check that package.json version is in versions.json
-    var currentVersion = require('../package.json').version;
-    if (sortedVersions.indexOf(currentVersion) === -1) {
-        die('package.json version is not present in versions.json');
-    }
-}
-
-if (process.argv.length === 3) {
-    verify(process.argv[2]);
-    process.exit(0);
-} else {
-    console.log('verify.js <versions_file>');
-}
-
-
-

release/versions-stable.json

@@ -1,7 +0,0 @@
-{
-    "0.0.1": {
-        "revision": "9f09f8e7a8633e8b3341bb9c610f5f631ccd288c",
-        "imageId": 7531071,
-        "next": null
-    }
-}

run.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-
-echo
-echo "Starting Cloudron at port 443"
-echo
-
-readonly BOX_SRC_DIR="$(cd $(dirname "$0"); pwd)"
-readonly NGINX_ROOT=~/.yellowtent/nginx
-readonly PROVISION_VERSION=0.1
-readonly PROVISION_BOX_VERSIONS_URL=0.1
-readonly DATA_DIR=~/.yellowtent/data
-readonly FQDN=admin-localhost
-
-mkdir -p "${NGINX_ROOT}/applications"
-mkdir -p "${NGINX_ROOT}/cert"
-mkdir -p "${DATA_DIR}"
-
-# get the database current
-npm run-script migrate
-
-cp setup/start/nginx/nginx.conf "${NGINX_ROOT}/nginx.conf"
-cp setup/start/nginx/mime.types "${NGINX_ROOT}/mime.types"
-cp setup/start/nginx/cert/* "${NGINX_ROOT}/cert/"
-
-# adjust the generated nginx config for local use
-touch "${NGINX_ROOT}/naked_domain.conf"
-sed -e "s/##ADMIN_FQDN##/${FQDN}/" -e "s|##BOX_SRC_DIR##|${BOX_SRC_DIR}|" setup/start/nginx/admin.conf_template > "${NGINX_ROOT}/applications/admin.conf"
-sed -e "s/user www-data/user ${USER}/" -i "${NGINX_ROOT}/nginx.conf"
-
-# add webadmin oauth client
-readonly WEBADMIN_ID=abcdefg
-readonly WEBADMIN_SCOPES="root,profile,users,apps,settings,roleAdmin"
-sqlite3 "${DATA_DIR}/cloudron.sqlite" "INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES (\"${WEBADMIN_ID}\", \"webadmin\", \"cid-webadmin\", \"secret-webadmin\", \"WebAdmin\", \"https://${FQDN}\", \"${WEBADMIN_SCOPES}\")"
-
-# start nginx
-sudo nginx -c nginx.conf -p "${NGINX_ROOT}"
-

scripts/checkInstall

@@ -1,40 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
-
-# reset sudo timestamp to avoid wrong success
-sudo -k || sudo --reset-timestamp
-
-# checks if all scripts are sudo access
-scripts=("${SOURCE_DIR}/src/scripts/rmappdir.sh" \
-         "${SOURCE_DIR}/src/scripts/reloadnginx.sh" \
-         "${SOURCE_DIR}/src/scripts/backup.sh" \
-         "${SOURCE_DIR}/src/scripts/reboot.sh" \
-         "${SOURCE_DIR}/src/scripts/reloadcollectd.sh")
-
-for script in "${scripts[@]}"; do
-    if [[ $(sudo -n "${script}" --check 2>/dev/null) != "OK" ]]; then
-        echo ""
-        echo "${script} does not have sudo access."
-        echo "You have to add the lines below to /etc/sudoers.d/yellowtent."
-        echo ""
-        echo "Defaults!${script} env_keep=HOME"
-        echo "${USER} ALL=(ALL) NOPASSWD: ${script}"
-        echo ""
-        exit 1
-    fi
-done
-
-if ! docker inspect girish/test:0.6 >/dev/null 2>/dev/null; then
-    echo "docker pull girish/test:0.6 for tests to run"
-    exit 1
-fi
-
-if ! docker inspect girish/redis:0.1 >/dev/null 2>/dev/null; then
-    echo "docker pull girish/redis:0.1 for tests to run"
-    exit 1
-fi
-
-exit 0

scripts/createReleaseTarball

@@ -0,0 +1,123 @@
+#!/bin/bash
+
+set -eu
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revision:,output:,publish,no-upload" -n "$0" -- "$@")
+eval set -- "${args}"
+
+readonly RELEASE_TOOL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../release" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+delete_bundle="yes"
+commitish="HEAD"
+publish="no"
+upload="yes"
+bundle_file=""
+
+while true; do
+    case "$1" in
+    --revision) commitish="$2"; shift 2;;
+    --output) bundle_file="$2"; delete_bundle="no"; shift 2;;
+    --no-upload) upload="no"; shift;;
+    --publish) publish="yes"; shift;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+if [[ "${upload}" == "no" && "${publish}" == "yes" ]]; then
+    echo "Cannot publish without uploading"
+    exit 1
+fi
+
+readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint?
+
+assertNotEmpty AWS_DEV_ACCESS_KEY
+assertNotEmpty AWS_DEV_SECRET_KEY
+
+if ! $(cd "${SOURCE_DIR}" && git diff --exit-code >/dev/null); then
+    echo "You have local changes, stash or commit them to proceed"
+    exit 1
+fi
+
+if [[ "$(node --version)" != "v4.1.1" ]]; then
+    echo "This script requires node 4.1.1"
+    exit 1
+fi
+
+version=$(cd "${SOURCE_DIR}" && git rev-parse "${commitish}")
+bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
+[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${version}.tar.gz"
+
+chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
+echo "Checking out code [${version}] into ${bundle_dir}"
+(cd "${SOURCE_DIR}" && git archive --format=tar ${version} | (cd "${bundle_dir}" && tar xf -))
+
+if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then
+    echo "Reusing dev modules from cache"
+    cp -r "${TMPDIR}/boxtarball.cache/node_modules-all/." "${bundle_dir}/node_modules"
+else
+    echo "Installing modules with dev dependencies"
+    (cd "${bundle_dir}" && npm install)
+
+    echo "Caching dev dependencies"
+    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-all"
+    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-all/"
+    cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all"
+fi
+
+echo "Building webadmin assets"
+(cd "${bundle_dir}" && gulp)
+
+echo "Remove intermediate files required at build-time only"
+rm -rf "${bundle_dir}/node_modules/"
+rm -rf "${bundle_dir}/webadmin/src"
+rm -rf "${bundle_dir}/gulpfile.js"
+
+if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then
+    echo "Reusing prod modules from cache"
+    cp -r "${TMPDIR}/boxtarball.cache/node_modules-prod/." "${bundle_dir}/node_modules"
+else
+    echo "Installing modules for production"
+    (cd "${bundle_dir}" && npm install --production --no-optional)
+
+    echo "Caching prod dependencies"
+    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-prod"
+    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-prod/"
+    cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod"
+fi
+
+echo "Create final tarball"
+(cd "${bundle_dir}" && tar czf "${bundle_file}" .)
+echo "Cleaning up ${bundle_dir}"
+rm -rf "${bundle_dir}"
+
+if [[ "${upload}" == "yes" ]]; then
+    echo "Uploading bundle to S3"
+    # That special header is needed to allow access with singed urls created with different aws credentials than the ones the file got uploaded
+    s3cmd --multipart-chunk-size-mb=5 --ssl --acl-public --access_key="${AWS_DEV_ACCESS_KEY}" --secret_key="${AWS_DEV_SECRET_KEY}" --no-mime-magic put "${bundle_file}" "s3://dev-cloudron-releases/box-${version}.tar.gz"
+
+    versions_file_url="https://dev-cloudron-releases.s3.amazonaws.com/box-${version}.tar.gz"
+    echo "The URL for the versions file is: ${versions_file_url}"
+
+    if [[ "${publish}" == "yes" ]]; then
+        echo "Publishing to dev"
+        ${RELEASE_TOOL_DIR}/release create --env dev --code "${versions_file_url}"
+    fi
+fi
+
+if [[ "${delete_bundle}" == "no" ]]; then
+    echo "Tarball preserved at ${bundle_file}"
+else
+    rm "${bundle_file}"
+fi
+

scripts/createSourceTarball

@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-[[ ! -f "${HOME}/.s3cfg" ]] && echo "~/.s3cfg missing" && exit 1
-
-# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
-# brew install gnu-getopt to get the GNU getopt on OS X
-[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-args=$(${GNU_GETOPT} -o "" -l "revision:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-commitish="HEAD"
-
-while true; do
-    case "$1" in
-    --revision) commitish="$2"; shift 2;;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-done
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint?
-
-version=$(cd "${SOURCE_DIR}" && git rev-parse "${commitish}")
-bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
-bundle_file="${TMPDIR}/box-${version}.tar.gz"
-
-chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
-echo "Checking out code [${version}] into ${bundle_dir}"
-(cd "${SOURCE_DIR}" && git archive --format=tar HEAD | (cd "${bundle_dir}" && tar xf -))
-
-echo "Installing modules"
-cd "${bundle_dir}" && npm install --production
-
-cd "${bundle_dir}" && tar czvf "${bundle_file}" .
-
-echo "Uploading bundle to S3"
-${SOURCE_DIR}/node_modules/.bin/s3-cli put --acl-public "${bundle_file}" "s3://cloudron-releases/box-${version}.tar.gz"
-
-echo "Cleaning up ${bundle_dir}"
-rm -rf "${bundle_dir}" "${bundle_file}"
-

setup/DESIGN.md

@@ -0,0 +1,57 @@
+This document gives the design of this setup code.
+
+box code should be delivered in the form of a (docker) container.
+This is not the case currently but we want to do structure the code
+in spirit that way.
+
+### container.sh
+This contains code that essential goes into Dockerfile.
+
+This file contains static configuration over a base image. Currently,
+the yellowtent user is created in the installer base image but it
+could very well be placed here.
+
+The idea is that the installer would simply remove the old box container
+and replace it with a new one for an update.
+
+Because we do not package things as Docker yet, we should be careful
+about the code here. We have to expect remains of an older setup code.
+For example, older systemd or nginx configs might be around.
+
+The config directory is _part_ of the container and is not a VOLUME.
+Which is to say that the files will be nuked from one update to the next.
+
+The data directory is a VOLUME. Contents of this directory are expected
+to survive an update. This is a good place to place config files that
+are "dynamic" and need to survive restarts. For example, the infra
+version (see below) or the mysql/postgresql data etc.
+
+### start.sh
+  * It is called in 3 modes - new, update, restore.
+
+  * The first thing this does is to do the static container.sh setup.
+
+  * It then downloads any box restore data and restores the box db from the
+    backup.
+
+  * It then proceeds to call the db-migrate script.
+
+  * It then does dynamic configuration like setting up nginx, collectd.
+
+  * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
+
+  * box services are then started
+
+setup_infra.sh
+This setups containers like graphite, mail and the addons containers.
+
+Containers are relaunched based on the INFRA_VERSION. The script compares
+the version here with the version in the file DATA_DIR/INFRA_VERSION.
+
+If they match, the containers are not recreated and nothing is to be done.
+nginx, collectd configs are part of data already and containers are running.
+
+If they do not match, it deletes all containers (including app containers) and starts
+them all afresh. Important thing here is that, DATA_DIR is never removed across
+updates. So, it is only the containers being recreated and not the data.
+

setup/INFRA_VERSION

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# If you change the infra version, be sure to put a warning
+# in the change log
+
+INFRA_VERSION=21
+
+# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+# These constants are used in the installer script as well
+BASE_IMAGE=cloudron/base:0.8.0
+MYSQL_IMAGE=cloudron/mysql:0.8.0
+POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
+MONGODB_IMAGE=cloudron/mongodb:0.8.0
+REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
+MAIL_IMAGE=cloudron/mail:0.9.0
+GRAPHITE_IMAGE=cloudron/graphite:0.8.0
+
+MYSQL_REPO=cloudron/mysql
+POSTGRESQL_REPO=cloudron/postgresql
+MONGODB_REPO=cloudron/mongodb
+REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
+MAIL_REPO=cloudron/mail
+GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -3,34 +3,77 @@
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 json="${script_dir}/../node_modules/.bin/json"
 
+# IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
+arg_api_server_origin=""
 arg_box_versions_url=""
+arg_fqdn=""
+arg_is_custom_domain="false"
+arg_restore_key=""
+arg_restore_url=""
+arg_retire="false"
+arg_tls_config=""
 arg_tls_cert=""
 arg_tls_key=""
-arg_app_server_url=""
-arg_fqdn=""
 arg_token=""
 arg_version=""
-arg_is_custom_domain="false"
+arg_web_server_origin=""
+arg_backup_config=""
+arg_dns_config=""
+arg_provider=""
 
-args=$(getopt -o "" -l "boxversionsurl:,data:,version:" -n "$0" -- "$@")
+args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
-    --boxversionsurl) arg_box_versions_url="$2";;
+    --retire)
+        arg_retire="true"
+        shift
+        ;;
     --data)
-        read -r arg_app_server_url arg_fqdn arg_token arg_is_custom_domain <<EOF
-        $(echo "$2" | $json appServerUrl fqdn token isCustomDomain | tr '\n' ' ')
+        # only read mandatory non-empty parameters here
+        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_is_custom_domain arg_box_versions_url arg_version <<EOF
+        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
 EOF
+        # read possibly empty parameters here
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
+        arg_token=$(echo "$2" | $json token)
+        arg_provider=$(echo "$2" | $json provider)
+
+        arg_tls_config=$(echo "$2" | $json tlsConfig)
+        [[ "${arg_tls_config}" == "null" ]] && arg_tls_config=""
+
+        arg_restore_url=$(echo "$2" | $json restore.url)
+        [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
+
+        arg_restore_key=$(echo "$2" | $json restore.key)
+        [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
+
+        arg_backup_config=$(echo "$2" | $json backupConfig)
+        [[ "${arg_backup_config}" == "null" ]] && arg_backup_config=""
+
+        arg_dns_config=$(echo "$2" | $json dnsConfig)
+        [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
+
+        shift 2
         ;;
-    --version) arg_version="$2";;
     --) break;;
     *) echo "Unknown option $1"; exit 1;;
     esac
-
-    shift 2
 done
 
-
+echo "Parsed arguments:"
+echo "api server: ${arg_api_server_origin}"
+echo "box versions url: ${arg_box_versions_url}"
+echo "fqdn: ${arg_fqdn}"
+echo "custom domain: ${arg_is_custom_domain}"
+echo "restore key: ${arg_restore_key}"
+echo "restore url: ${arg_restore_url}"
+echo "tls cert: ${arg_tls_cert}"
+echo "tls key: ${arg_tls_key}"
+echo "token: ${arg_token}"
+echo "tlsConfig: ${arg_tls_config}"
+echo "version: ${arg_version}"
+echo "web server: ${arg_web_server_origin}"
+echo "provider: ${arg_provider}"

setup/container.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# This file can be used in Dockerfile
+
+readonly container_files="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/container"
+
+readonly CONFIG_DIR="/home/yellowtent/configs"
+readonly DATA_DIR="/home/yellowtent/data"
+
+########## create config directory
+rm -rf "${CONFIG_DIR}"
+sudo -u yellowtent mkdir "${CONFIG_DIR}"
+
+########## systemd
+rm -f /etc/systemd/system/janitor.*
+cp -r "${container_files}/systemd/." /etc/systemd/system/
+systemctl daemon-reload
+systemctl enable cloudron.target
+
+########## sudoers
+rm -f /etc/sudoers.d/yellowtent
+cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
+
+########## collectd
+rm -rf /etc/collectd
+ln -sfF "${DATA_DIR}/collectd" /etc/collectd
+
+########## apparmor docker profile
+cp "${container_files}/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
+systemctl restart apparmor
+
+########## nginx
+# link nginx config to system config
+unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
+ln -s "${DATA_DIR}/nginx" /etc/nginx
+
+########## mysql
+cp "${container_files}/mysql.cnf" /etc/mysql/mysql.cnf
+
+########## Enable services
+update-rc.d -f collectd defaults
+

setup/container/docker-cloudron-app.apparmor

@@ -0,0 +1,32 @@
+#include <tunables/global>
+
+
+profile docker-cloudron-app flags=(attach_disconnected,mediate_deleted) {
+
+  #include <abstractions/base>
+
+  ptrace peer=@{profile_name},
+
+  network,
+  capability,
+  file,
+  umount,
+
+  deny @{PROC}/sys/fs/** wklx,
+  deny @{PROC}/sysrq-trigger rwklx,
+  deny @{PROC}/mem rwklx,
+  deny @{PROC}/kmem rwklx,
+  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
+  deny @{PROC}/sys/kernel/*/** wklx,
+
+  deny mount,
+
+  deny /sys/[^f]*/** wklx,
+  deny /sys/f[^s]*/** wklx,
+  deny /sys/fs/[^c]*/** wklx,
+  deny /sys/fs/c[^g]*/** wklx,
+  deny /sys/fs/cg[^r]*/** wklx,
+  deny /sys/firmware/efi/efivars/** rwklx,
+  deny /sys/kernel/security/** rwklx,
+}
+

setup/container/mysql.cnf

@@ -0,0 +1,7 @@
+!includedir /etc/mysql/conf.d/
+!includedir /etc/mysql/mysql.conf.d/
+
+# http://bugs.mysql.com/bug.php?id=68514
+[mysqld]
+performance_schema=OFF
+max_connection=50

setup/container/sudoers

@@ -0,0 +1,32 @@
+# sudo logging breaks journalctl output with very long urls (systemd bug)
+Defaults !syslog
+
+Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupbox.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restoreapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
+
+Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh

setup/container/systemd/box.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+StartLimitInterval=1
+StartLimitBurst=60
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smart Cloud
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service
+After=box.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/splash/nginx/mime.types

@@ -1,80 +0,0 @@
-
-types {
-    text/html                             html htm shtml;
-    text/css                              css;
-    text/xml                              xml;
-    image/gif                             gif;
-    image/jpeg                            jpeg jpg;
-    application/x-javascript              js;
-    application/atom+xml                  atom;
-    application/rss+xml                   rss;
-
-    text/mathml                           mml;
-    text/plain                            txt;
-    text/vnd.sun.j2me.app-descriptor      jad;
-    text/vnd.wap.wml                      wml;
-    text/x-component                      htc;
-
-    image/png                             png;
-    image/tiff                            tif tiff;
-    image/vnd.wap.wbmp                    wbmp;
-    image/x-icon                          ico;
-    image/x-jng                           jng;
-    image/x-ms-bmp                        bmp;
-    image/svg+xml                         svg svgz;
-    image/webp                            webp;
-
-    application/java-archive              jar war ear;
-    application/mac-binhex40              hqx;
-    application/msword                    doc;
-    application/pdf                       pdf;
-    application/postscript                ps eps ai;
-    application/rtf                       rtf;
-    application/vnd.ms-excel              xls;
-    application/vnd.ms-powerpoint         ppt;
-    application/vnd.wap.wmlc              wmlc;
-    application/vnd.google-earth.kml+xml  kml;
-    application/vnd.google-earth.kmz      kmz;
-    application/x-7z-compressed           7z;
-    application/x-cocoa                   cco;
-    application/x-java-archive-diff       jardiff;
-    application/x-java-jnlp-file          jnlp;
-    application/x-makeself                run;
-    application/x-perl                    pl pm;
-    application/x-pilot                   prc pdb;
-    application/x-rar-compressed          rar;
-    application/x-redhat-package-manager  rpm;
-    application/x-sea                     sea;
-    application/x-shockwave-flash         swf;
-    application/x-stuffit                 sit;
-    application/x-tcl                     tcl tk;
-    application/x-x509-ca-cert            der pem crt;
-    application/x-xpinstall               xpi;
-    application/xhtml+xml                 xhtml;
-    application/zip                       zip;
-
-    application/octet-stream              bin exe dll;
-    application/octet-stream              deb;
-    application/octet-stream              dmg;
-    application/octet-stream              eot;
-    application/octet-stream              iso img;
-    application/octet-stream              msi msp msm;
-
-    audio/midi                            mid midi kar;
-    audio/mpeg                            mp3;
-    audio/ogg                             ogg;
-    audio/x-m4a                           m4a;
-    audio/x-realaudio                     ra;
-
-    video/3gpp                            3gpp 3gp;
-    video/mp4                             mp4;
-    video/mpeg                            mpeg mpg;
-    video/quicktime                       mov;
-    video/webm                            webm;
-    video/x-flv                           flv;
-    video/x-m4v                           m4v;
-    video/x-mng                           mng;
-    video/x-ms-asf                        asx asf;
-    video/x-ms-wmv                        wmv;
-    video/x-msvideo                       avi;
-}

setup/splash/nginx/nginx.conf

@@ -1,63 +0,0 @@
-user www-data;
-worker_processes 1;
-pid /run/nginx.pid;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include mime.types;
-    default_type application/octet-stream;
-
-    access_log /var/log/nginx/access.log;
-    error_log /var/log/nginx/error.log;
-
-    root ##SETUP_WEBSITE_DIR##;
-    index index.html;
-
-    server {
-        listen 80;
-
-        location / {
-            # redirect everything to HTTPS
-            return 301 https://$host$request_uri;
-        }
-    }
-
-    # HTTPS server
-    server {
-        listen 443;
-
-        error_page 503 /index.html;
-
-        location /index.html {
-            # allow access to this page
-            add_header Cache-Control no-cache;
-        }
-
-        location /3rdparty/bootstrap.min.css {
-            # allow access to this page
-            add_header Cache-Control no-cache;
-        }
-
-        location /progress.json {
-            # allow access to this page
-            add_header Cache-Control no-cache;
-        }
-
-        location / {
-            return 503;
-        }
-
-        ssl on;
-        ssl_certificate      cert/host.cert;
-        ssl_certificate_key  cert/host.key;
-
-        ssl_session_timeout 5m;
-
-        ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-        ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
-        ssl_prefer_server_ciphers on;
-    }
-}

setup/splash/website/index.html

@@ -1,36 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
-
-    <title> Cloudron Webadmin </title>
-
-    <link href="3rdparty/bootstrap.min.css" rel="stylesheet">
-</head>
-
-<body>
-    <!-- Modal update progress -->
-    <div class="modal show" id="updateProgressModal" tabindex="-1" role="dialog" aria-labelledby="updateProgressModalLabel" aria-hidden="true" data-keyboard    ="false" data-backdrop="static">
-        <div class="modal-dialog">
-            <div class="modal-content">
-                <div class="modal-header">
-                    <h4 class="modal-title" id="updateProgressModalLabel">Update in progress...</h4>
-                </div>
-                <div class="modal-body">
-                    <div class="progress progress-striped active">
-                        <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="100" aria-valuemin="0" aria-valuemax="100" style="width: 100%"></div>
-                    </div>
-                </div>
-            </div>
-            <!-- /.modal-content -->
-        </div>
-        <!-- /.modal-dialog -->
-    </div>
-
-    <script>
-        setTimeout(location.reload.bind(location, true /* forceGet from server */), 10000);
-    </script>
-</body>
-
-</html>

setup/splashpage.sh

@@ -1,32 +1,40 @@
 #!/bin/bash
 
-set -eu
+set -eu -o pipefail
 
-readonly NGINX_CONFIG_DIR="/home/yellowtent/setup/configs/nginx" # do not reuse configs since it will be removed by installer
 readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly BOX_SRC_DIR="/home/yellowtent/box"
+readonly DATA_DIR="/home/yellowtent/data"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
 
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
 # copy the website
 rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-rm -rf "${NGINX_CONFIG_DIR}" && mkdir -p "${NGINX_CONFIG_DIR}"
-sed -e "s|##SETUP_WEBSITE_DIR##|${SETUP_WEBSITE_DIR}|" "${script_dir}/splash/nginx/nginx.conf" > "${NGINX_CONFIG_DIR}/nginx.conf"
-cp "${script_dir}/splash/nginx/mime.types" "${NGINX_CONFIG_DIR}/mime.types"
-mkdir -p "${NGINX_CONFIG_DIR}/cert"
-echo "${arg_tls_cert}" > "${NGINX_CONFIG_DIR}/cert/host.cert"
-echo "${arg_tls_key}" > "${NGINX_CONFIG_DIR}/cert/host.key"
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+    rm -f ${DATA_DIR}/nginx/applications/*
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+else
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+fi
 
-# link in the new nginx config
-unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
-ln -s "${NGINX_CONFIG_DIR}" /etc/nginx
-
-touch "${SETUP_WEBSITE_DIR}/progress.json"
+echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
 
 nginx -s reload

setup/start.sh

@@ -1,196 +1,213 @@
 #!/bin/bash
 
-# Count installer files so that we can correlate install and postinstall logs
-install_count=$(find /var/log/cloudron -name "installer*" | wc -l)
-exec > >(tee "/var/log/cloudron/start-$install_count.log")
-exec 2>&1
-
-set -eux
+set -eu -o pipefail
 
 echo "==== Cloudron Start ===="
 
 readonly USER="yellowtent"
-# NOTE: Do NOT use BOX_SRC_DIR for accessing code and config files. This script will be run from a temp directory
-# and the whole code will relocated to BOX_SRC_DIR by the installer. Use paths relative to script_dir or box_src_tmp_dir
 readonly BOX_SRC_DIR="/home/${USER}/box"
 readonly DATA_DIR="/home/${USER}/data"
 readonly CONFIG_DIR="/home/${USER}/configs"
-readonly MAIL_SERVER_IP="172.17.120.120" # hardcoded in haraka container
 readonly SETUP_PROGRESS_JSON="/home/yellowtent/setup/website/progress.json"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-box_src_tmp_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
-admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "admin.${arg_fqdn}" ||  echo "admin-${arg_fqdn}")
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_origin="https://${admin_fqdn}"
+
+readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
 
 set_progress() {
-    local progress="$1"
+    local percent="$1"
     local message="$2"
 
-    echo "==== ${message} ===="
-    (echo "{ \"progress\": \"${progress}\", \"message\": \"${message}\" }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
+    echo "==== ${percent} - ${message} ===="
+    (echo "{ \"update\": { \"percent\": \"${percent}\", \"message\": \"${message}\" }, \"backup\": {} }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
 }
 
-set_progress "5" "Configuring Sudoers file"
-cat > /etc/sudoers.d/yellowtent <<EOF
-Defaults!${BOX_SRC_DIR}/src/scripts/rmappdir.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/rmappdir.sh
+set_progress "1" "Create container"
+$script_dir/container.sh
 
-Defaults!${BOX_SRC_DIR}/src/scripts/reloadnginx.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reloadnginx.sh
+set_progress "10" "Ensuring directories"
+# keep these in sync with paths.js
+[[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
+mkdir -p "${DATA_DIR}/box/appicons"
+mkdir -p "${DATA_DIR}/box/certs"
+mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/box/acme" # acme keys
+mkdir -p "${DATA_DIR}/graphite"
 
-Defaults!${BOX_SRC_DIR}/src/scripts/backup.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/backup.sh
+mkdir -p "${DATA_DIR}/mysql"
+mkdir -p "${DATA_DIR}/postgresql"
+mkdir -p "${DATA_DIR}/mongodb"
+mkdir -p "${DATA_DIR}/snapshots"
+mkdir -p "${DATA_DIR}/addons"
+mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${DATA_DIR}/acme" # acme challenges
 
-Defaults!${BOX_SRC_DIR}/src/scripts/reboot.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reboot.sh
+# bookkeep the version as part of data
+echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
 
-Defaults!${BOX_SRC_DIR}/src/scripts/reloadcollectd.sh env_keep=HOME
-${USER} ALL=(root) NOPASSWD: ${BOX_SRC_DIR}/src/scripts/reloadcollectd.sh
+# remove old snapshots. if we do want to keep this around, we will have to fix the chown -R below
+# which currently fails because these are readonly fs
+echo "Cleaning up snapshots"
+find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
 
-EOF
+# restart mysql to make sure it has latest config
+service mysql restart
 
-set_progress "10" "Migrating data"
+readonly mysql_root_password="password"
+mysqladmin -u root -ppassword password password # reset default root password
+mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
+
+if [[ -n "${arg_restore_url}" ]]; then
+    set_progress "15" "Downloading restore data"
+
+    echo "Downloading backup: ${arg_restore_url} and key: ${arg_restore_key}"
+
+    while true; do
+        if $curl -L "${arg_restore_url}" | openssl aes-256-cbc -d -pass "pass:${arg_restore_key}" | tar -zxf - -C "${DATA_DIR}/box"; then break; fi
+        echo "Failed to download data, trying again"
+    done
+
+    set_progress "21" "Setting up MySQL"
+    if [[ -f "${DATA_DIR}/box/box.mysqldump" ]]; then
+        echo "Importing existing database into MySQL"
+        mysql -u root -p${mysql_root_password} box < "${DATA_DIR}/box/box.mysqldump"
+    fi
+fi
+
+set_progress "25" "Migrating data"
 sudo -u "${USER}" -H bash <<EOF
-set -eux
-cd "${box_src_tmp_dir}"
-PATH="${PATH}:${box_src_tmp_dir}/node_modules/.bin" npm run-script migrate_data
+set -eu
+cd "${BOX_SRC_DIR}"
+BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
 EOF
 
-set_progress "15" "Setup nginx"
-nginx_config_dir="${CONFIG_DIR}/nginx"
-nginx_appconfig_dir="${CONFIG_DIR}/nginx/applications"
+set_progress "28" "Setup collectd"
+cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
+## collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
+#mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
+## detect device, let it fail if non exists
+#[[ -b "/dev/vda1" ]] && disk_device="/dev/vda1"
+#[[ -b "/dev/xvda1" ]] && disk_device="/dev/xvda1"
+#vda1_id=$(blkid -s UUID -o value ${disk_device})
+#ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
+service collectd restart
 
-# copy nginx config
-mkdir -p "${nginx_appconfig_dir}"
-cp "${script_dir}/start/nginx/nginx.conf" "${nginx_config_dir}/nginx.conf"
-cp "${script_dir}/start/nginx/mime.types" "${nginx_config_dir}/mime.types"
-touch "${nginx_config_dir}/naked_domain.conf"
-sed -e "s/##ADMIN_FQDN##/${admin_fqdn}/" -e "s|##BOX_SRC_DIR##|${BOX_SRC_DIR}|" "${script_dir}/start/nginx/admin.conf_template" > "${nginx_appconfig_dir}/admin.conf"
+set_progress "30" "Setup nginx"
+mkdir -p "${DATA_DIR}/nginx/applications"
+cp "${script_dir}/start/nginx/nginx.conf" "${DATA_DIR}/nginx/nginx.conf"
+cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"
 
-certificate_dir="${nginx_config_dir}/cert"
-mkdir -p "${certificate_dir}"
-echo "${arg_tls_cert}" > ${certificate_dir}/host.cert
-echo "${arg_tls_key}" > ${certificate_dir}/host.key
+# generate these for update code paths as well to overwrite splash
+admin_cert_file="${DATA_DIR}/nginx/cert/host.cert"
+admin_key_file="${DATA_DIR}/nginx/cert/host.key"
+if [[ -f "${DATA_DIR}/box/certs/${admin_fqdn}.cert" && -f "${DATA_DIR}/box/certs/${admin_fqdn}.key" ]]; then
+    admin_cert_file="${DATA_DIR}/box/certs/${admin_fqdn}.cert"
+    admin_key_file="${DATA_DIR}/box/certs/${admin_fqdn}.key"
+fi
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
-# link nginx config to system config
-unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
-ln -s "${nginx_config_dir}" /etc/nginx
-
-chown "${USER}:${USER}" -R "/home/${USER}"
-
-set_progress "20" "Removing existing container"
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
+mkdir -p "${DATA_DIR}/nginx/cert"
+if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
+    cp "${DATA_DIR}/box/certs/host.cert" "${DATA_DIR}/nginx/cert/host.cert"
+    cp "${DATA_DIR}/box/certs/host.key" "${DATA_DIR}/nginx/cert/host.key"
+else
+    echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
+    echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
 fi
 
-set_progress "30" "Setup collectd and graphite"
-${script_dir}/start/setup_collectd.sh
+set_progress "33" "Changing ownership"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}"
 
-set_progress "40" "Setup haraka mail relay"
-docker rm -f haraka || true
-docker pull girish/haraka:0.1 || true # this line is for dev convenience since it's already part of base image
-haraka_container_id=$(docker run --restart=always -d --name="haraka" --cap-add="NET_ADMIN"\
-    -p 127.0.0.1:25:25 \
-    -h "${arg_fqdn}" \
-    -e "DOMAIN_NAME=${arg_fqdn}" \
-    -v "${CONFIG_DIR}/haraka:/app/data" \
-    girish/haraka:0.1)
-echo "Haraka container id: ${haraka_container_id}"
-# Every docker restart results in a new IP. Give our mail server a
-# static IP. Alternately, we need to link the mail container with
-# all our apps
-# This IP is set by the haraka container on every start and the firewall
-# allows connect to port 25. The ping gets the ARP lookup working
-echo "Checking connectivity to haraka(${MAIL_SERVER_IP})"
-if ! ping -c 20 "${MAIL_SERVER_IP}"; then
-    echo "Could not connect to mail server"
-fi
+set_progress "40" "Setting up infra"
+${script_dir}/start/setup_infra.sh "${arg_fqdn}"
 
-set_progress "50" "Setup MySQL addon"
-docker rm -f mysql || true
-mysql_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-docker pull girish/mysql:0.1 || true # this line for dev convenience since it's already part of base image
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -p 127.0.0.1:3306:3306 \
-    -h "${arg_fqdn}" \
-    -e "MYSQL_ROOT_PASSWORD=${mysql_root_password}" \
-    -e "MYSQL_ROOT_HOST=${docker0_ip}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    girish/mysql:0.1)
-echo "MySQL container id: ${mysql_container_id}"
-
-set_progress "60" "Setup Postgres addon"
-docker rm -f postgresql || true
-postgresql_root_password=$(pwgen -1 -s)
-docker pull girish/postgresql:0.1 || true # this line for dev convenience since it's already part of base image
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -p 127.0.0.1:5432:5432 \
-    -h "${arg_fqdn}" \
-    -e "POSTGRESQL_ROOT_PASSWORD=${postgresql_root_password}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/mysql" \
-    girish/postgresql:0.1)
-echo "PostgreSQL container id: ${postgresql_container_id}"
-
-set_progress "70" "Pulling Redis addon"
-docker pull girish/redis:0.1 || true # this line for dev convenience since it's already part of base image
-
-set_progress "80" "Creating cloudron.conf"
-cloudron_sqlite="${DATA_DIR}/cloudron.sqlite"
-admin_origin="https://${admin_fqdn}"
+set_progress "65" "Creating cloudron.conf"
 sudo -u yellowtent -H bash <<EOF
-set -eux
+set -eu
 echo "Creating cloudron.conf"
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
     "version": "${arg_version}",
     "token": "${arg_token}",
-    "appServerUrl": "${arg_app_server_url}",
+    "apiServerOrigin": "${arg_api_server_origin}",
+    "webServerOrigin": "${arg_web_server_origin}",
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "mailServer": "${MAIL_SERVER_IP}",
-    "mailUsername": "admin@${arg_fqdn}",
-    "addons": {
-        "mysql": {
-            "rootPassword": "${mysql_root_password}"
-        },
-        "postgresql": {
-            "rootPassword": "${postgresql_root_password}"
-        }
+    "adminEmail": "admin@${arg_fqdn}",
+    "provider": "${arg_provider}",
+    "database": {
+        "hostname": "localhost",
+        "username": "root",
+        "password": "${mysql_root_password}",
+        "port": 3306,
+        "name": "box"
     }
 }
 CONF_END
 
-echo "Marking apps for restore"
-# TODO: do not auto-start stopped containers (httpPort might need fixing to start them)
-sqlite3 "${cloudron_sqlite}" 'UPDATE apps SET installationState = "pending_restore", healthy = NULL, runState = NULL, containerId = NULL, httpPort = NULL, installationProgress = NULL'
-
-# Add webadmin oauth client
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,profile,users,apps,settings,roleAdmin"
-ADMIN_ID=$(cat /proc/sys/kernel/random/uuid)
-sqlite3 "${cloudron_sqlite}" "INSERT OR REPLACE INTO clients (id, appId, clientId, clientSecret, name, redirectURI, scope) VALUES (\"\$ADMIN_ID\", \"webadmin\", \"cid-webadmin\", \"secret-webadmin\", \"WebAdmin\", \"${admin_origin}\", \"\$ADMIN_SCOPES\")"
-
+echo "Creating config.json for webadmin"
+cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
+{
+    "webServerOrigin": "${arg_web_server_origin}"
+}
+CONF_END
 EOF
 
-# bookkeep the version as part of data
-echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/version"
+# Add Backup Configuration
+if [[ ! -z "${arg_backup_config}" ]]; then
+    echo "Add Backup Config"
 
-set_progress "90" "Setup supervisord"
-${script_dir}/start/setup_supervisord.sh
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
+fi
 
-set_progress "95" "Reloading supervisor"
-${script_dir}/start/reload_supervisord.sh
+# Add DNS Configuration
+if [[ ! -z "${arg_dns_config}" ]]; then
+    echo "Add DNS Config"
 
-set_progress "99" "Reloading nginx"
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
+fi
+
+# Add TLS Configuration
+if [[ ! -z "${arg_tls_config}" ]]; then
+    echo "Add TLS Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
+fi
+
+# Add webadmin oauth client
+# The domain might have changed, therefor we have to update the record
+# !!! This needs to be in sync with the webadmin, specifically login_callback.js
+echo "Add webadmin oauth cient"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+
+echo "Add localhost test oauth client"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+
+set_progress "80" "Starting Cloudron"
+systemctl start cloudron.target
+
+sleep 2 # give systemd sometime to start the processes
+
+set_progress "85" "Reloading nginx"
 nginx -s reload
 
 set_progress "100" "Done"

setup/start/collectd.conf

@@ -52,20 +52,20 @@ Interval 20
 # accessed.                                                                  #
 ##############################################################################
 
-#LoadPlugin logfile
-LoadPlugin syslog
+LoadPlugin logfile
+#LoadPlugin syslog
 
-#<Plugin logfile>
-#   LogLevel "info"
-#   File STDOUT
-#   Timestamp true
-#   PrintSeverity false
-#</Plugin>
-
-<Plugin syslog>
-    LogLevel info
+<Plugin logfile>
+   LogLevel "info"
+   File "/var/log/collectd.log"
+   Timestamp true
+   PrintSeverity false
 </Plugin>
 
+#<Plugin syslog>
+#    LogLevel info
+#</Plugin>
+
 ##############################################################################
 # LoadPlugin section                                                         #
 #----------------------------------------------------------------------------#
@@ -96,7 +96,7 @@ LoadPlugin df
 #LoadPlugin entropy
 #LoadPlugin ethstat
 #LoadPlugin exec
-LoadPlugin filecount
+#LoadPlugin filecount
 #LoadPlugin fscache
 #LoadPlugin gmond
 #LoadPlugin hddtemp
@@ -193,12 +193,11 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin df>
-   Device "/dev/vda1"
-   Device "/dev/loop0"
-   Device "/dev/loop1"
+   FSType "tmpfs"
+   MountPoint "/dev"
 
    ReportByDevice true
-   IgnoreSelected false
+   IgnoreSelected true
 
    ValuesAbsolute true
    ValuesPercentage true
@@ -221,7 +220,7 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin processes>
-   ProcessMatch "app" "node app.js"
+   ProcessMatch "app" "node box.js"
 </Plugin>
 
 <Plugin swap>

setup/start/nginx/admin.conf_template

@@ -1,33 +0,0 @@
-server {
-    listen       443;
-    server_name  ##ADMIN_FQDN##;
-
-    ssl                  on;
-    # paths are relative to prefix and not to this file
-    ssl_certificate      cert/host.cert;
-    ssl_certificate_key  cert/host.key;
-
-
-    ssl_session_timeout  5m;
-
-    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
-    ssl_prefer_server_ciphers on;
-
-    location /api/ {
-        proxy_pass   http://127.0.0.1:3000;
-        client_max_body_size 1m;
-    }
-
-    # graphite paths
-    location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
-        proxy_pass   http://127.0.0.1:8000;
-        client_max_body_size 1m;
-    }
-
-    location / {
-        root   ##BOX_SRC_DIR##/webadmin/dist/;
-        index  index.html index.htm;
-    }
-}
-

setup/start/nginx/appconfig.ejs

@@ -0,0 +1,114 @@
+# http://nginx.org/en/docs/http/websocket.html
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+server {
+    listen       443;
+    server_name  <%= vhost %>;
+
+    ssl                  on;
+    # paths are relative to prefix and not to this file
+    ssl_certificate      <%= certFilePath %>;
+    ssl_certificate_key  <%= keyFilePath %>;
+    ssl_session_timeout  5m;
+    ssl_session_cache shared:SSL:50m;
+
+    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
+    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
+    # https://cipherli.st/
+    ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
+
+    proxy_http_version 1.1;
+    proxy_intercept_errors on;
+    proxy_read_timeout       3500;
+    proxy_connect_timeout    3250;
+
+    proxy_set_header   Host               $host;
+    proxy_set_header   X-Forwarded-For    $remote_addr;
+    proxy_set_header   X-Forwarded-Host   $host;
+    proxy_set_header   X-Forwarded-Proto  https;
+
+    # upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    # only serve up the status page if we get proxy gateway errors
+    error_page 502 503 504 @appstatus;
+    location @appstatus {
+        return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
+    }
+
+    location / {
+        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+        proxy_buffer_size       128k;
+        proxy_buffers           4 256k;
+        proxy_busy_buffers_size 256k;
+
+        # Disable check to allow unlimited body sizes
+        client_max_body_size 0;
+
+<% if ( endpoint === 'admin' ) { %>
+        location /api/ {
+            proxy_pass   http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
+
+        # graphite paths
+        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+            proxy_pass   http://127.0.0.1:8000;
+            client_max_body_size 1m;
+        }
+
+        location / {
+            root   <%= sourceDir %>/webadmin/dist;
+            index  index.html index.htm;
+        }
+
+<% } else if ( endpoint === 'oauthproxy' ) { %>
+        proxy_pass http://127.0.0.1:3003;
+        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
+<% } else if ( endpoint === 'app' ) { %>
+        proxy_pass http://127.0.0.1:<%= port %>;
+<% } else if ( endpoint === 'splash' ) { %>
+        root   <%= sourceDir %>;
+
+        error_page 503 /update.html;
+
+        location /update.html {
+            add_header Cache-Control no-cache;
+        }
+
+        location /theme.css {
+            add_header Cache-Control no-cache;
+        }
+
+        location /3rdparty/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /js/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /progress.json {
+            add_header Cache-Control no-cache;
+        }
+
+        location /api/v1/cloudron/progress {
+            add_header Cache-Control no-cache;
+            default_type application/json;
+            alias <%= sourceDir %>/progress.json;
+        }
+
+        location / {
+            return 503;
+        }
+<% } %>
+    }
+}
+

setup/start/nginx/nginx.conf

@@ -17,6 +17,9 @@ http {
         '"$request" $status $body_bytes_sent $request_time '
         '"$http_referer" "$http_user_agent"';
 
+    # required for long host names
+    server_names_hash_bucket_size 128;
+
     access_log access.log combined2;
 
     sendfile        on;
@@ -35,6 +38,12 @@ http {
             deny all;
         }
 
+        # acme challenges
+        location /.well-known/acme-challenge/ {
+            default_type text/plain;
+            alias /home/yellowtent/data/acme/;
+        }
+
         location / {
             # redirect everything to HTTPS
             return 301 https://$host$request_uri;
@@ -49,11 +58,16 @@ http {
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
+        error_page 404 = @fallback;
+        location @fallback {
+            internal;
+            root /home/yellowtent/box/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
         return 404;
     }
 
-    include naked_domain.conf;
-
     include applications/*.conf;
 }
 

setup/start/reload_supervisord.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# looks like restarting supervisor completely is the only way to reload it
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
-

setup/start/setup_collectd.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly GRAPHITE_DIR="/home/yellowtent/data/graphite"
-readonly COLLECTD_CONFIG_DIR="/home/yellowtent/configs/collectd"
-readonly COLLECTD_APPCONFIG_DIR="${COLLECTD_CONFIG_DIR}/collectd.conf.d"
-
-readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-mkdir -p "${GRAPHITE_DIR}"
-
-docker rm -f graphite || true
-docker pull girish/graphite:0.2
-docker run --restart=always -d --name="graphite" \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${GRAPHITE_DIR}:/app/data" girish/graphite:0.2
-
-mkdir -p "${COLLECTD_APPCONFIG_DIR}"
-cp -r "${script_dir}/collectd/collectd.conf" "${COLLECTD_CONFIG_DIR}/collectd.conf"
-rm -rf /etc/collectd
-ln -sfF "${COLLECTD_CONFIG_DIR}" /etc/collectd
-chown -R yellowtent.yellowtent "${COLLECTD_CONFIG_DIR}"
-
-update-rc.d -f collectd defaults
-/etc/init.d/collectd restart
-

setup/start/setup_infra.sh

@@ -0,0 +1,129 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly DATA_DIR="/home/yellowtent/data"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
+
+arg_fqdn="$1"
+
+# removing containers ensures containers are launched with latest config updates
+# restore code in appatask does not delete old containers
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
+    echo "Infrastructure is upto date"
+    exit 0
+fi
+
+echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
+
+existing_containers=$(docker ps -qa)
+echo "Remove containers: ${existing_containers}"
+if [[ -n "${existing_containers}" ]]; then
+    echo "${existing_containers}" | xargs docker rm -f
+fi
+
+# graphite
+graphite_container_id=$(docker run --restart=always -d --name="graphite" \
+    -m 75m \
+    --memory-swap 150m \
+    -p 127.0.0.1:2003:2003 \
+    -p 127.0.0.1:2004:2004 \
+    -p 127.0.0.1:8000:8000 \
+    -v "${DATA_DIR}/graphite:/app/data" \
+    --read-only -v /tmp -v /run \
+    "${GRAPHITE_IMAGE}")
+echo "Graphite container id: ${graphite_container_id}"
+if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old graphite images"
+fi
+
+# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
+mail_container_id=$(docker run --restart=always -d --name="mail" \
+    -m 75m \
+    --memory-swap 150m \
+    -h "${arg_fqdn}" \
+    -e "DOMAIN_NAME=${arg_fqdn}" \
+    -v "${DATA_DIR}/box/mail:/app/data" \
+    --read-only -v /tmp -v /run \
+    "${MAIL_IMAGE}")
+echo "Mail container id: ${mail_container_id}"
+if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mail images"
+fi
+
+# mysql
+mysql_addon_root_password=$(pwgen -1 -s)
+docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
+cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
+readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
+readonly MYSQL_ROOT_HOST='${docker0_ip}'
+EOF
+mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -m 100m \
+    --memory-swap 200m \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
+    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
+    "${MYSQL_IMAGE}")
+echo "MySQL container id: ${mysql_container_id}"
+if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mysql images"
+fi
+
+# postgresql
+postgresql_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
+readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
+EOF
+postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -m 100m \
+    --memory-swap 200m \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
+    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
+    "${POSTGRESQL_IMAGE}")
+echo "PostgreSQL container id: ${postgresql_container_id}"
+if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old postgresql images"
+fi
+
+# mongodb
+mongodb_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
+readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
+EOF
+mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -m 100m \
+    --memory-swap 200m \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
+    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
+    "${MONGODB_IMAGE}")
+echo "Mongodb container id: ${mongodb_container_id}"
+if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mongodb images"
+fi
+
+# redis
+if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old redis images"
+fi
+
+# only touch apps in installed state. any other state is just resumed by the taskmanager
+if [[ "${infra_version}" == "none" ]]; then
+    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
+    echo "Marking installed apps for restore"
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
+else
+    # if existing infra was found, just mark apps for reconfiguration
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
+fi
+
+echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"

setup/start/setup_supervisord.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-readonly BOX_SRC_DIR="/home/yellowtent/box"
-readonly DATA_DIR="/home/yellowtent/data"
-
-readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-rm -rf /etc/supervisor
-mkdir -p /etc/supervisor/conf.d
-cp "${script_dir}/supervisord/supervisord.conf" /etc/supervisor/
-
-echo "Writing supervisor configs..."
-
-cat > /etc/supervisor/conf.d/box.conf <<EOF
-[program:box]
-command=/usr/bin/node "${BOX_SRC_DIR}/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",NODE_ENV="cloudron"
-EOF
-
-cat > /etc/supervisor/conf.d/oauthproxy.conf <<EOF
-[program:oauthproxy]
-command=/usr/bin/node "${BOX_SRC_DIR}/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/proxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
-EOF
-
-cat > /etc/supervisor/conf.d/apphealthtask.conf <<EOF
-[program:apphealthtask]
-command=/usr/bin/node "${BOX_SRC_DIR}/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
-EOF
-

setup/start/supervisord/supervisord.conf

@@ -1,33 +0,0 @@
-; supervisor config file
-
-; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[supervisord]
-logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
-pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-nodaemon = false
-childlogdir = /var/log/supervisor/
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=http://127.0.0.1:9001
-
-; The [include] section can just contain the "files" setting.  This
-; setting can list multiple files (separated by whitespace or
-; newlines).  It can also contain wildcards.  The filenames are
-; interpreted as relative to this file.  Included files *cannot*
-; include files themselves.
-
-[include]
-files = conf.d/*.conf
-

setup/stop.sh

@@ -1,15 +1,7 @@
 #!/bin/bash
 
-set -eu
+set -eu -o pipefail
 
-echo "Stopping box code"
-
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
+echo "Stopping cloudron"
 
+systemctl stop cloudron.target

src/appdb.js

@@ -2,25 +2,17 @@
 
 'use strict';
 
-var assert = require('assert'),
-    async = require('async'),
-    database = require('./database.js'),
-    DatabaseError = require('./databaseerror'),
-    debug = require('debug')('box:appdb'),
-    safe = require('safetydance'),
-    util = require('util');
-
 exports = module.exports = {
     get: get,
     getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
+    getByContainerId: getByContainerId,
     add: add,
     exists: exists,
     del: del,
     update: update,
     getAll: getAll,
     getPortBindings: getPortBindings,
-    clear: clear,
 
     setAddonConfig: setAddonConfig,
     getAddonConfig: getAddonConfig,
@@ -31,179 +23,220 @@ exports = module.exports = {
     setHealth: setHealth,
     setInstallationCommand: setInstallationCommand,
     setRunCommand: setRunCommand,
-    getAppVersions: getAppVersions,
+    getAppStoreIds: getAppStoreIds,
 
-    // status codes
-    ISTATE_PENDING_INSTALL: 'pending_install',
-    ISTATE_PENDING_CONFIGURE: 'pending_configure',
-    ISTATE_PENDING_UNINSTALL: 'pending_uninstall',
-    ISTATE_PENDING_RESTORE: 'pending_restore',
-    ISTATE_PENDING_UPDATE: 'pending_update',
-    ISTATE_ERROR: 'error',
-    ISTATE_INSTALLED: 'installed',
+    // installation codes (keep in sync in UI)
+    ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
+    ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
+    ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
+    ISTATE_PENDING_UPDATE: 'pending_update', // update from installed state preserving data
+    ISTATE_PENDING_FORCE_UPDATE: 'pending_force_update', // update from any state preserving data
+    ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app
+    ISTATE_ERROR: 'error', // error executing last pending_* command
+    ISTATE_INSTALLED: 'installed', // app is installed
 
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
-    RSTATE_STOPPED: 'stopped', // app stopped by user
-    RSTATE_DEAD: 'dead', // app stopped on it's own
-    RSTATE_ERROR: 'error'
+    RSTATE_STOPPED: 'stopped', // app stopped by use
+
+    // run codes (keep in sync in UI)
+    HEALTH_HEALTHY: 'healthy',
+    HEALTH_UNHEALTHY: 'unhealthy',
+    HEALTH_ERROR: 'error',
+    HEALTH_DEAD: 'dead',
+
+    _clear: clear
 };
 
-var APPS_FIELDS = [ 'id', 'appStoreId', 'version', 'installationState', 'installationProgress', 'runState',
-    'healthy', 'containerId', 'manifestJson', 'httpPort', 'location', 'dnsRecordId', 'accessRestriction' ].join(',');
+var assert = require('assert'),
+    async = require('async'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror'),
+    safe = require('safetydance'),
+    util = require('util');
 
-var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.version', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
-    'apps.healthy', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId', 'apps.accessRestriction' ].join(',');
+var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.oauthProxy' ].join(',');
 
-var PORT_BINDINGS_FIELDS = [ 'hostPort', 'containerPort', 'appId' ].join(',');
+var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
 function postProcess(result) {
-    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
+    assert.strictEqual(typeof result, 'object');
 
+    assert(result.manifestJson === null || typeof result.manifestJson === 'string');
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
+    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
+    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
+    delete result.lastBackupConfigJson;
+
+    assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
+    result.oldConfig = safe.JSON.parse(result.oldConfigJson);
+    delete result.oldConfigJson;
+
     assert(result.hostPorts === null || typeof result.hostPorts === 'string');
-    assert(result.containerPorts === null || typeof result.containerPorts === 'string');
+    assert(result.environmentVariables === null || typeof result.environmentVariables === 'string');
 
     result.portBindings = { };
     var hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
-    var containerPorts = result.containerPorts === null ? [ ] : result.containerPorts.split(',');
+    var environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
 
     delete result.hostPorts;
-    delete result.containerPorts;
+    delete result.environmentVariables;
 
-    for (var i = 0; i < hostPorts.length; i++) {
-        result.portBindings[containerPorts[i]] = hostPorts[i];
+    for (var i = 0; i < environmentVariables.length; i++) {
+        result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
     }
+
+    result.oauthProxy = !!result.oauthProxy;
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
 }
 
 function get(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
-        + ' FROM apps LEFT OUTER JOIN appPortBindings WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE apps.id = ? GROUP BY apps.id', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        postProcess(result[0]);
 
-        postProcess(result);
-
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
 function getBySubdomain(subdomain, callback) {
-    assert(typeof subdomain === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE location = ? GROUP BY apps.id', [ subdomain ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        postProcess(result[0]);
 
-        postProcess(result);
-
-        callback(null, result);
+        callback(null, result[0]);
     });
 }
 
 function getByHttpPort(httpPort, callback) {
-    assert(typeof httpPort === 'number');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof httpPort, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
-        + '  FROM apps LEFT OUTER JOIN appPortBindings WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE httpPort = ? GROUP BY apps.id', [ httpPort ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        if (typeof result === 'undefined') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        postProcess(result[0]);
 
-        postProcess(result);
+        callback(null, result[0]);
+    });
+}
 
-        callback(null, result);
+function getByContainerId(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
     });
 }
 
 function getAll(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT ' + APPS_FIELDS_PREFIXED + ','
-        + 'GROUP_CONCAT(appPortBindings.hostPort) AS hostPorts, GROUP_CONCAT(appPortBindings.containerPort) AS containerPorts'
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
         + ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId'
         + ' GROUP BY apps.id ORDER BY apps.id', function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        if (typeof results === 'undefined') results = [ ];
-
         results.forEach(postProcess);
 
         callback(null, results);
     });
 }
 
-function add(id, appStoreId, location, portBindings, accessRestriction, callback) {
-    assert(typeof id === 'string');
-    assert(typeof appStoreId === 'string');
-    assert(typeof location === 'string');
-    assert(typeof portBindings === 'object');
-    assert(typeof accessRestriction === 'string');
-    assert(typeof callback === 'function');
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, oauthProxy, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof appStoreId, 'string');
+    assert(manifest && typeof manifest === 'object');
+    assert.strictEqual(typeof manifest.version, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof oauthProxy, 'boolean');
+    assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
-    var conn = database.beginTransaction();
+    var manifestJson = JSON.stringify(manifest);
+    var accessRestrictionJson = JSON.stringify(accessRestriction);
 
-    conn.run('INSERT INTO apps (id, appStoreId, installationState, location, accessRestriction) VALUES (?, ?, ?, ?, ?)',
-           [ id, appStoreId, exports.ISTATE_PENDING_INSTALL, location, accessRestriction ], function (error) {
-        if (error || !this.lastID) database.rollback(conn);
+    var queries = [ ];
+    queries.push({
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, oauthProxy) VALUES (?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, oauthProxy ]
+    });
 
-        if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-
-        if (error || !this.lastID) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        async.eachSeries(Object.keys(portBindings), function iterator(containerPort, callback) {
-            conn.run('INSERT INTO appPortBindings (hostPort, containerPort, appId) VALUES (?, ?, ?)',
-                     [ portBindings[containerPort], containerPort, id ], callback);
-        }, function done(error) {
-            if (error) database.rollback(conn);
-
-            if (error && error.code === 'SQLITE_CONSTRAINT') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-
-            if (error /* || !this.lastID*/) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-            database.commit(conn, callback);
+    Object.keys(portBindings).forEach(function (env) {
+        queries.push({
+            query: 'INSERT INTO appPortBindings (environmentVariable, hostPort, appId) VALUES (?, ?, ?)',
+            args: [ env, portBindings[env], id ]
         });
     });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
 }
 
 function exists(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.get('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
+    database.query('SELECT 1 FROM apps WHERE id=?', [ id ], function (error, result) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null, typeof result !== 'undefined');
+        return callback(null, result.length !== 0);
     });
 }
 
 function getPortBindings(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
+    database.query('SELECT ' + PORT_BINDINGS_FIELDS + ' FROM appPortBindings WHERE appId = ?', [ id ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        results = results || [ ];
         var portBindings = { };
         for (var i = 0; i < results.length; i++) {
-            portBindings[results[i].containerPort] = results[i].hostPort;
+            portBindings[results[i].environmentVariable] = results[i].hostPort;
         }
 
         callback(null, portBindings);
@@ -211,29 +244,29 @@ function getPortBindings(id, callback) {
 }
 
 function del(id, callback) {
-    assert(typeof id === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    var conn = database.beginTransaction();
-    conn.run('DELETE FROM appPortBindings WHERE appId = ?', [ id ], function (error) {
-        conn.run('DELETE FROM apps WHERE id = ?', [ id ], function (error) {
-            if (error || this.changes !== 1) database.rollback(conn);
+    var queries = [
+        { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] },
+        { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] }
+    ];
 
-            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-            if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    database.transaction(queries, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-            database.commit(conn, callback);
-        });
+        callback(null);
     });
 }
 
 function clear(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     async.series([
-        database.run.bind(null, 'DELETE FROM appPortBindings'),
-        database.run.bind(null, 'DELETE FROM apps'),
-        database.run.bind(null, 'DELETE FROM appAddonConfigs')
+        database.query.bind(null, 'DELETE FROM appPortBindings'),
+        database.query.bind(null, 'DELETE FROM appAddonConfigs'),
+        database.query.bind(null, 'DELETE FROM apps')
     ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         return callback(null);
@@ -241,150 +274,158 @@ function clear(callback) {
 }
 
 function update(id, app, callback) {
-    updateWithConstraints(id, app, callback);
+    updateWithConstraints(id, app, '', callback);
 }
 
 function updateWithConstraints(id, app, constraints, callback) {
-    assert(typeof id === 'string');
-    assert(typeof app === 'object');
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof constraints, 'string');
+    assert.strictEqual(typeof callback, 'function');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
 
-    if (typeof constraints === 'function') {
-        callback = constraints;
-        constraints = '';
-    } else {
-        assert(typeof constraints === 'string');
-        assert(typeof callback === 'function');
-    }
+    var queries = [ ];
 
-    var portBindings = app.portBindings || { };
-
-    var conn = database.beginTransaction();
-    async.eachSeries(Object.keys(portBindings), function iterator(containerPort, callback) {
-        var values = [ portBindings[containerPort], containerPort, id ];
-        conn.run('UPDATE appPortBindings SET hostPort = ? WHERE containerPort = ? AND appId = ?', values, callback);
-    }, function seriesDone(error) {
-        if (error) {
-            database.rollback(conn);
-            return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        }
-
-        var args = [ ], values = [ ];
-        for (var p in app) {
-            if (!app.hasOwnProperty(p)) continue;
-
-            if (p === 'manifest') {
-                args.push('manifestJson = ?');
-                values.push(JSON.stringify(app[p]));
-            } else if (p !== 'portBindings') {
-                args.push(p + ' = ?');
-                values.push(app[p]);
-            }
-        }
-
-        if (values.length === 0) return database.commit(conn, callback);
-
-        values.push(id);
-
-        conn.run('UPDATE apps SET ' + args.join(', ') + ' WHERE id = ? ' + constraints, values, function (error) {
-            if (error || this.changes !== 1) database.rollback(conn);
-            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-            if (this.changes !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-            database.commit(conn, callback);
+    if ('portBindings' in app) {
+        var portBindings = app.portBindings || { };
+        // replace entries by app id
+        queries.push({ query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] });
+        Object.keys(portBindings).forEach(function (env) {
+            var values = [ portBindings[env], env, id ];
+            queries.push({ query: 'INSERT INTO appPortBindings (hostPort, environmentVariable, appId) VALUES(?, ?, ?)', args: values });
         });
-    });
-}
-
-// sets health on installed apps that have a runState which is not null or pending
-function setHealth(appId, healthy, runState, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof healthy === 'boolean');
-    assert(typeof runState === 'string');
-    assert(typeof callback === 'function');
-
-    var values = {
-        healthy: healthy,
-        runState: runState
-    };
-
-    var constraints = 'AND runState NOT GLOB "pending_*" AND installationState = "installed"';
-    if (runState === exports.RSTATE_DEAD) { // don't mark stopped apps as dead
-        constraints += ' AND runState != "stopped"';
     }
 
-    updateWithConstraints(appId, values, constraints, callback);
-}
-
-function setInstallationCommand(appId, installationState, values, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof installationState === 'string');
-
-    if (typeof values === 'function') {
-        callback = values;
-        values = { };
-    } else {
-        assert(typeof values === 'object');
-        assert(typeof callback === 'function');
+    var fields = [ ], values = [ ];
+    for (var p in app) {
+        if (p === 'manifest') {
+            fields.push('manifestJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'lastBackupConfig') {
+            fields.push('lastBackupConfigJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'oldConfig') {
+            fields.push('oldConfigJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p === 'accessRestriction') {
+            fields.push('accessRestrictionJson = ?');
+            values.push(JSON.stringify(app[p]));
+        } else if (p !== 'portBindings') {
+            fields.push(p + ' = ?');
+            values.push(app[p]);
+        }
     }
 
-    values.installationState = installationState;
-
-    if (installationState === exports.ISTATE_PENDING_UNINSTALL) {
-        updateWithConstraints(appId, values, '', callback);
-    } else {
-        updateWithConstraints(appId, values, 'AND installationState NOT GLOB "pending_*"', callback);
+    if (values.length !== 0) {
+        values.push(id);
+        queries.push({ query: 'UPDATE apps SET ' + fields.join(', ') + ' WHERE id = ? ' + constraints, args: values });
     }
-}
 
-function setRunCommand(appId, runState, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof runState === 'string');
-    assert(typeof callback === 'function');
-
-    var values = { runState: runState };
-    updateWithConstraints(appId, values, 'AND runState NOT GLOB "pending_*" AND installationState = "installed"', callback);
-}
-
-function getAppVersions(callback) {
-    assert(typeof callback === 'function');
-
-    database.all('SELECT id, appStoreId, version FROM apps', function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        results = results || [ ];
-        callback(null, results);
-    });
-}
-
-function setAddonConfig(appId, addonId, env, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof addonId === 'string');
-    assert(util.isArray(env));
-    assert(typeof callback === 'function');
-
-    if (env.length === 0) return callback(null);
-
-    var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
-    var args = [ ], queryArgs = [ ];
-    for (var i = 0; i < env.length; i++) {
-        args.push(appId, addonId, env[i]);
-        queryArgs.push('(?, ?, ?)');
-    }
-
-    database.run(query + queryArgs.join(','), args, function (error) {
+    database.transaction(queries, function (error, results) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results[results.length - 1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         return callback(null);
     });
 }
 
-function unsetAddonConfig(appId, addonId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof addonId === 'string');
-    assert(typeof callback === 'function');
+// not sure if health should influence runState
+function setHealth(appId, health, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
+    var values = { health: health };
+
+    var constraints = 'AND runState NOT LIKE "pending_%" AND installationState = "installed"';
+
+    updateWithConstraints(appId, values, constraints, callback);
+}
+
+function setInstallationCommand(appId, installationState, values, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof installationState, 'string');
+
+    if (typeof values === 'function') {
+        callback = values;
+        values = { };
+    } else {
+        assert.strictEqual(typeof values, 'object');
+        assert.strictEqual(typeof callback, 'function');
+    }
+
+    values.installationState = installationState;
+    values.installationProgress = '';
+
+    // Rules are:
+    // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
+    // restore is allowed from installed or error state
+    // update and configure are allowed only in installed state
+
+    if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
+        updateWithConstraints(appId, values, '', callback);
+    } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
+        updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
+        updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else {
+        callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
+    }
+}
+
+function setRunCommand(appId, runState, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof runState, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var values = { runState: runState };
+    updateWithConstraints(appId, values, 'AND runState NOT LIKE "pending_%" AND installationState = "installed"', callback);
+}
+
+function getAppStoreIds(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT id, appStoreId FROM apps', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAddonConfig(appId, addonId, env, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert(util.isArray(env));
+    assert.strictEqual(typeof callback, 'function');
+
+    unsetAddonConfig(appId, addonId, function (error) {
+        if (error) return callback(error);
+
+        if (env.length === 0) return callback(null);
+
+        var query = 'INSERT INTO appAddonConfigs(appId, addonId, value) VALUES ';
+        var args = [ ], queryArgs = [ ];
+        for (var i = 0; i < env.length; i++) {
+            args.push(appId, addonId, env[i]);
+            queryArgs.push('(?, ?, ?)');
+        }
+
+        database.query(query + queryArgs.join(','), args, function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            return callback(null);
+        });
+    });
+}
+
+function unsetAddonConfig(appId, addonId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -392,10 +433,10 @@ function unsetAddonConfig(appId, addonId, callback) {
 }
 
 function unsetAddonConfigByAppId(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.run('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
+    database.query('DELETE FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -403,29 +444,29 @@ function unsetAddonConfigByAppId(appId, callback) {
 }
 
 function getAddonConfig(appId, addonId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof addonId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof addonId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, result) {
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ? AND addonId = ?', [ appId, addonId ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         var config = [ ];
-        result.forEach(function (v) { config.push(v.value); });
+        results.forEach(function (v) { config.push(v.value); });
 
         callback(null, config);
     });
 }
 
 function getAddonConfigByAppId(appId, callback) {
-    assert(typeof appId === 'string');
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-    database.all('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, result) {
+    database.query('SELECT value FROM appAddonConfigs WHERE appId = ?', [ appId ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         var config = [ ];
-        result.forEach(function (v) { config.push(v.value); });
+        results.forEach(function (v) { config.push(v.value); });
 
         callback(null, config);
     });

src/apphealthmonitor.js

@@ -0,0 +1,195 @@
+'use strict';
+
+var appdb = require('./appdb.js'),
+    assert = require('assert'),
+    async = require('async'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:apphealthmonitor'),
+    docker = require('./docker.js').connection,
+    mailer = require('./mailer.js'),
+    superagent = require('superagent'),
+    util = require('util');
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
+var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var gHealthInfo = { }; // { time, emailSent }
+var gRunTimeout = null;
+var gDockerEventStream = null;
+
+function debugApp(app) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? app.location : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function setHealth(app, health, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof health, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+
+    if (!(app.id in gHealthInfo)) { // add new apps to list
+        gHealthInfo[app.id] = { time: now, emailSent: false };
+    }
+
+    if (health === appdb.HEALTH_HEALTHY) {
+        gHealthInfo[app.id].time = now;
+    } else if (Math.abs(now - gHealthInfo[app.id].time) > UNHEALTHY_THRESHOLD) {
+        if (gHealthInfo[app.id].emailSent) return callback(null);
+
+        debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
+
+        if (app.appStoreId !== '') mailer.appDied(app); // do not send mails for dev apps
+        gHealthInfo[app.id].emailSent = true;
+    } else {
+        debugApp(app, 'waiting for sometime to update the app health');
+        return callback(null);
+    }
+
+    appdb.setHealth(app.id, health, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled?
+        if (error) return callback(error);
+
+        app.health = health;
+
+        callback(null);
+    });
+}
+
+
+// callback is called with error for fatal errors and not if health check failed
+function checkAppHealth(app, callback) {
+    if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
+        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
+        return callback(null);
+    }
+
+    var container = docker.getContainer(app.containerId),
+        manifest = app.manifest;
+
+    container.inspect(function (err, data) {
+        if (err || !data || !data.State) {
+            debugApp(app, 'Error inspecting container');
+            return setHealth(app, appdb.HEALTH_ERROR, callback);
+        }
+
+        if (data.State.Running !== true) {
+            debugApp(app, 'exited');
+            return setHealth(app, appdb.HEALTH_DEAD, callback);
+        }
+
+        // poll through docker network instead of nginx to bypass any potential oauth proxy
+        var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
+        superagent
+            .get(healthCheckUrl)
+            .redirects(0)
+            .timeout(HEALTHCHECK_INTERVAL)
+            .end(function (error, res) {
+            if (error && !error.response) {
+                debugApp(app, 'not alive (network error): %s', error.message);
+                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
+            } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
+                debugApp(app, 'not alive : %s', error || res.status);
+                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
+            } else {
+                setHealth(app, appdb.HEALTH_HEALTHY, callback);
+            }
+        });
+    });
+}
+
+function processApps(callback) {
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(error);
+
+        async.each(apps, checkAppHealth, function (error) {
+            if (error) console.error(error);
+
+            var alive = apps
+                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
+                .map(function (a) { return a.location; }).join(', ');
+
+            debug('apps alive: [%s]', alive);
+
+            callback(null);
+        });
+    });
+}
+
+function run() {
+    processApps(function (error) {
+        if (error) console.error(error);
+
+        gRunTimeout = setTimeout(run, HEALTHCHECK_INTERVAL);
+    });
+}
+
+/*
+    OOM can be tested using stress tool like so:
+        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
+        apt-get update && apt-get install stress
+        stress --vm 1 --vm-bytes 200M --vm-hang 0
+*/
+function processDockerEvents() {
+    // note that for some reason, the callback is called only on the first event
+    debug('Listening for docker events');
+    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return console.error(error);
+
+        gDockerEventStream = stream;
+
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            var ev = JSON.parse(data);
+            debug('Container ' + ev.id + ' went OOM');
+            appdb.getByContainerId(ev.id, function (error, app) {
+                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
+                var context = JSON.stringify(ev);
+                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
+
+                debug('OOM Context: %s', context);
+
+                // do not send mails for dev apps
+                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+            });
+        });
+
+        stream.on('error', function (error) {
+            console.error('Error reading docker events', error);
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+
+        stream.on('end', function () {
+            console.error('Docker event stream ended');
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+    });
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting apphealthmonitor');
+
+    processDockerEvents();
+
+    run();
+
+    callback();
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    clearTimeout(gRunTimeout);
+    gDockerEventStream.end();
+
+    callback();
+}