up link is relative

For custom domains, we do not set the A record for the naked domain
(because the user might be using it for his own). This means that
a:domain.com will not work.

The solution is to simply use the admin domain.

.gitattributes

@@ -1,6 +1,7 @@
-# Skip files when using git archive
+# following files are skipped when exporting using git archive
+/release export-ignore
+/admin export-ignore
+test export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
-/scripts export-ignore
-test export-ignore
 

.gitignore

@@ -3,11 +3,9 @@ coverage/
 docs/
 webadmin/dist/
 setup/splash/website/
+installer/src/certs/server.key
 
-# vim swam files
+# vim swap files
 *.swp
 
-# supervisor
-supervisord.pid
-supervisord.log
 

CHANGES

@@ -0,0 +1,441 @@
+[0.0.1]
+- Hot Chocolate
+
+[0.0.2]
+- Hotfix appstore ui in webadim
+
+[0.0.3]
+- Tall Pike
+
+[0.0.4]
+- This will be 0.0.4 changes
+
+[0.0.5]
+- App install/configure route fixes
+
+[0.0.6]
+- Not sure what happenned here
+
+[0.0.7]
+- resetToken is now sent as part of create user
+- Same as 0.0.7 which got released by mistake
+
+[0.0.8]
+- Manifest changes
+
+[0.0.9]
+- Fix app restore
+- Fix backup issues
+
+[0.0.10]
+- Unknown orchestra
+
+[0.0.11]
+- Add ldap addon
+
+[0.0.12]
+- Support OAuth2 state
+
+[0.0.13]
+- Use docker image from cloudron repository
+
+[0.0.14]
+- Improve setup flow
+
+[0.0.15]
+- Improved Appstore view
+
+[0.0.16]
+- Improved Backup approach
+
+[0.0.17]
+- Upgrade testing
+- App auto updates
+- Usage graphs
+
+[0.0.18]
+- Rework backups and updates
+
+[0.0.19]
+- Graphite fixes
+- Avatar and Cloudron name support
+
+[0.0.20]
+- Apptask fixes
+- Chrome related fixes
+
+[0.0.21]
+- Increase nginx hostname size to 64
+
+[0.0.22]
+- Testing the e2e tests
+
+[0.0.23]
+- Better error status page
+- Fix updater and backup progress reporting
+- New avatar set
+- Improved setup wizard
+
+[0.0.24]
+- Hotfix the ldap support
+
+[0.0.25]
+- Add support page
+- Really fix ldap issues
+
+[0.0.26]
+- Add configurePath support
+
+[0.0.27]
+- Improved log collector
+
+[0.0.28]
+- Improve app feedback
+- Restyle login page
+
+[0.0.29]
+- Update to ubuntu 15.04
+
+[0.0.30]
+- Move to docker 1.7
+
+[0.0.31]
+- WARNING: This update restarts your containers
+- System processes are prioritized over apps
+- Add ldap group support
+
+[0.0.32]
+- MySQL addon update
+
+[0.0.33]
+- Fix graphs
+- Fix MySQL 5.6 memory usage
+
+[0.0.34]
+- Correctly mark apps pending for approval
+
+[0.0.35]
+- Fix ldap admin group username
+
+[0.0.36]
+- Fix restore without backup
+- Optimize image deletion during updates
+- Add memory accounting
+- Restrict access to metadata from containers
+
+[0.0.37]
+- Prepare for Selfhosting 1. part
+- Use userData instead of provisioning calls
+
+[0.0.38]
+- Account for Ext4 reserved block when partitioning disk
+
+[0.0.39]
+- Move subdomain management to the cloudron
+
+[0.0.40]
+- Add journal limit
+- Fix reprovisioning on reboot
+- Fix subdomain management during startup
+
+[0.0.41]
+- Finally bring things to a sane state
+
+[0.0.42]
+- Parallel apptask
+
+[0.0.43]
+- Move to systemd
+
+[0.0.44]
+- Fix apptask concurrency bug
+
+[0.0.45]
+- Retry subdomain registration
+
+[0.0.46]
+- Fix app update email notification
+
+[0.0.47]
+- Ensure box code quits within 5 seconds
+
+[0.0.48]
+- Styling fixes
+- Improved session handling
+
+[0.0.49]
+- Fix app autoupdate logic
+
+[0.0.50]
+- Use domainmanagement via CaaS
+
+[0.0.51]
+- Fix memory management
+
+[0.0.52]
+- Restrict addons memory
+- Get nofication about container OOMs
+
+[0.0.53]
+- Restrict addons memory
+- Get notification about container OOMs
+- Add retry to subdomain logic
+
+[0.0.54]
+- OAuth Proxy now uses internal port forwarding
+
+[0.0.55]
+- Setup cloudron timezone based on droplet region
+
+[0.0.56]
+- Use correct timezone in updater
+
+[0.0.57]
+- Fix systemd logging issues
+
+[0.0.58]
+- Ensure backups of failed apps are retained across archival cycles
+
+[0.0.59]
+- Installer API fixes
+
+[0.0.60]
+- Do full box backup on updates
+
+[0.0.61]
+- Track update notifications to inform admin only once
+
+[0.0.62]
+- Export bind dn and password from LDAP addon
+
+[0.0.63]
+- Fix creation of TXT records
+
+[0.0.64]
+- Stop apps in a retired cloudron
+- Retry downloading application on failure
+
+[0.0.65]
+- Do not send crash mails for apps in development
+
+[0.0.66]
+- Readonly application and addon containers
+
+[0.0.67]
+- Fix email notifications
+- Fix bug when restoring from certain backups
+
+[0.0.68]
+- Update graphite image
+- Add simpleauth addon support
+
+[0.0.69]
+- Support newer manifest format
+- Fix app listing rendering in chrome
+- Fix redis backup across upgrades
+
+[0.0.70]
+- Retry app download on error
+
+[0.0.71]
+- Fix oauth and simple auth login
+
+[0.0.72]
+- Cleanup application volumes periodically
+- New application logging design
+
+[0.0.73]
+- Update SSL certificate
+
+[0.0.74]
+- Support singleUser apps
+
+[0.0.75]
+- scheduler addon
+
+[0.0.76]
+- DNS Sync fixes
+- Show warning to user when memory limit reached
+
+[0.0.77]
+- Do not set hostname in app containers
+
+[0.0.78]
+- Support custom domains
+
+[0.0.79]
+- Move SSH Port
+
+[0.0.80]
+- Use journalctl for container logs
+
+[0.1.0]
+- Wait for configuration changes before starting Cloudron
+
+[0.1.1]
+- Ensure dns config for all cloudrons
+
+[0.1.2]
+- Make email work again
+- Add DKIM keys for custom domains
+
+[0.1.3]
+- Storage backend
+
+[0.1.4]
+- CaaS Backup configuration fix
+
+[0.1.5]
+- Use correct tokens for DNS backend
+
+[0.1.6]
+- Add hook to determine the api server of the box
+- Fix crash notification
+
+[0.2.0]
+- New cloudron exec implementation
+
+[0.2.1]
+- Update to node 4.1.1
+- Fix certification installation with custom domains
+
+[0.2.2]
+- Better debug output
+- Retry more times if docker registry goes down
+
+[0.3.0]
+- Update SSH keys
+- Allow bigger manifest files
+
+[0.4.0]
+- Update to docker 1.9.0
+
+[0.4.1]
+- Fix scheduler crash
+- Crucial OAuth fixes
+
+[0.4.2]
+- Fix crash when reporting backup error
+- Allow larger manifests
+
+[0.4.3]
+- Fix cloudron exec
+
+[0.4.4]
+- Initial Lets Encrypt integration
+
+[0.4.5]
+- Fixup nginx configuration to allow dynamic certificates
+
+[0.4.6]
+- LetsEncrypt integration for custom domains
+- Rate limit crash emails
+
+[0.5.0]
+- Enable staging Lets Encrypt Integration
+
+[0.5.1]
+- Display error dialog for app installation errors
+- Enable prod Lets Encrypt Integration
+- Handle apptask crashes correctly
+
+[0.5.2]
+- Fix apphealthtask crash
+- Use cgroup fs driver instead of systemd cgroup driver in docker
+
+[0.5.3]
+- Changes for e2e testing
+
+[0.5.4]
+- Fix bug in LE server selection
+
+[0.5.5]
+- Scheduler redesign
+- Fix journalctl logging
+
+[0.5.6]
+- Prepare for selfhosting option
+
+[0.5.7]
+- Move app images off the btrfs subvolume
+
+[0.6.0]
+- Consolidate code repositories
+
+[0.6.1]
+- Use no-reply as email from address for apps in naked domains
+- Update Lets Encrypt account with owner email when available
+- Fix email templates to indicate auto update
+- Add notification UI
+
+[0.6.2]
+- Fix `cloudron exec` container to have same namespaces as app
+- Add developmentMode to manifest
+
+[0.6.3]
+- Make sending invite for new users optional
+
+[0.6.4]
+- Add support for display names
+- Send invite links to admins for user setup
+- Enforce stronger passwords
+
+[0.6.5]
+- Finalize stronger password requirement
+
+[0.7.0]
+- Upgrade to 15.10
+- Do not remove docker images when in use by another container
+- Fix sporadic error when reconfiguring apps
+- Handle journald crashes gracefully
+
+[0.7.1]
+- Allow admins to edit users
+- Fix graphs
+- Support more LDAP cases
+- Allow appstore deep linking
+
+[0.7.2]
+- Fix 5xx errors when password does not meet requirements
+- Improved box update management using prereleases
+- Less aggressive disk space checks
+
+[0.8.0]
+- MySQL addon : multiple database support
+
+[0.8.1]
+- Set Host HTTP header when querying healthCheckPath
+- Show application Changelog in app update emails
+
+[0.9.0]
+- Fix bug in multdb mysql addon backup
+- Add initial user group support
+- Improved app memory limit handling
+
+[0.9.1]
+- Introduce per app group access control
+
+[0.9.2]
+- Fix bug where reconfiguring apps would trigger memory limit warning
+- Allow more apps to be installed in bigger sized cloudrons
+- Allow user to override memory limit warning and install anyway
+
+[0.9.3]
+- Admin flag is handled outside of groups
+- User interface fixes for groups
+- Allow to set access restrictions on app installation
+
+[0.10.0]
+- Upgrade to docker 1.10.2
+- Fix MySQL addon to handle heavier loads
+- Allow listing and download of backups (using the CLI tool)
+- Ubuntu security updates till 8th March 2016 (http://www.ubuntu.com/usn)
+
+[0.10.1]
+- Fix Let's Encrypt certificate renewal
+
+[0.10.2]
+- Apps can now bind with username or email with LDAP
+- Disallow updating an app with mismatching manifest id
+- Use admin domain instead of naked domain in the SPF record
+- Download Lets Encrypt intermediate cert

README.md

@@ -1,11 +1,17 @@
-The Box
+Cloudron a Smart Server
-=======
+=======================
 
-Development setup
------------------
-* sudo useradd -m yellowtent
-** This dummy user is required for supervisor 'box' configs
-** Add admin-localhost as 127.0.0.1 in /etc/hosts
-** All apps will be installed as hypened-subdomains of localhost. You should add
-   hyphened-subdomains of your apps into /etc/hosts
 
+
+Selfhost Instructions
+---------------------
+
+The smart server currently relies on an AWS account with access to Route53 and S3 and is tested on DigitalOcean and EC2.
+
+First create a virtual private server with Ubuntu 15.04 and run the following commands in an ssh session to initialize the base image:
+
+```
+curl https://s3.amazonaws.com/prod-cloudron-releases/installer.sh -o installer.sh
+chmod +x installer.sh
+./installer.sh <domain> <aws access key> <aws acccess secret> <backup bucket> <provider> <release sha1>
+```

app.js

@@ -1,47 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var server = require('./src/server.js'),
-    ldap = require('./src/ldap.js'),
-    config = require('./src/config.js');
-
-console.log();
-console.log('==========================================');
-console.log(' Cloudron will use the following settings ');
-console.log('==========================================');
-console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
-console.log(' Version:                        ', config.version());
-console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore token:                 ', config.token());
-console.log(' Appstore API server origin:     ', config.apiServerOrigin());
-console.log(' Appstore Web server origin:     ', config.webServerOrigin());
-console.log();
-console.log('==========================================');
-console.log();
-
-server.start(function (err) {
-    if (err) {
-        console.error('Error starting server', err);
-        process.exit(1);
-    }
-
-    console.log('Server listening on port ' + config.get('port'));
-
-    ldap.start(function (error) {
-        if (error) {
-            console.error('Error LDAP starting server', err);
-            process.exit(1);
-        }
-
-        console.log('LDAP server listen on port ' + config.get('ldapPort'));
-    });
-});
-
-var NOOP_CALLBACK = function () { };
-
-process.on('SIGINT', function () { server.stop(NOOP_CALLBACK); });
-process.on('SIGTERM', function () { server.stop(NOOP_CALLBACK); });

baseimage/createImage

@@ -0,0 +1,186 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+provider="digitalocean"
+installer_revision=$(git rev-parse HEAD)
+box_name=""
+server_id=""
+server_ip=""
+destroy_server="yes"
+deploy_env="dev"
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "provider:,revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --env) deploy_env="$2"; shift 2;;
+    --revision) installer_revision="$2"; shift 2;;
+    --provider) provider="$2"; shift 2;;
+    --name) box_name="$2"; destroy_server="no"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+echo "Creating image using ${provider}"
+if [[ "${provider}" == "digitalocean" ]]; then
+    if [[ "${deploy_env}" == "staging" ]]; then
+        assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
+        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
+    elif [[ "${deploy_env}" == "dev" ]]; then
+        assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
+        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
+    elif [[ "${deploy_env}" == "prod" ]]; then
+        assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
+        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
+    else
+        echo "No such env ${deploy_env}."
+        exit 1
+    fi
+
+    vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
+else
+    echo "Unknown provider : ${provider}"
+    exit 1
+fi
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}"
+readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${installer_revision}")
+
+if [[ -z "${box_name}" ]]; then
+    # if you change this, change the regexp is appstore/janitor.js
+    box_name="box-${deploy_env}-${pretty_revision}-${now}" # remove slashes
+
+    # create a new server if no name given
+    if ! caas_ssh_key_id=$($vps get_ssh_key_id "caas"); then
+        echo "Could not query caas ssh key"
+        exit 1
+    fi
+    echo "Detected caas ssh key id: ${caas_ssh_key_id}"
+
+    echo "Creating Server with name [${box_name}]"
+    if ! server_id=$($vps create ${caas_ssh_key_id} ${box_name}); then
+        echo "Failed to create server"
+        exit 1
+    fi
+    echo "Created server with id: ${server_id}"
+
+    # If we run scripts overenthusiastically without the wait, setup script randomly fails
+    echo -n "Waiting 120 seconds for server creation"
+    for i in $(seq 1 24); do
+        echo -n "."
+        sleep 5
+    done
+    echo ""
+else
+    if ! server_id=$($vps get_id "${box_name}"); then
+        echo "Could not determine id from name"
+        exit 1
+    fi
+    echo "Reusing server with id: ${server_id}"
+
+    $vps power_on "${server_id}"
+fi
+
+# Query until we get an IP
+while true; do
+    echo "Trying to get the server IP"
+    if server_ip=$($vps get_ip "${server_id}"); then
+        echo "Server IP : [${server_ip}]"
+        break
+    fi
+    echo "Timedout, trying again in 10 seconds"
+    sleep 10
+done
+
+while true; do
+    echo "Trying to copy init script to server"
+    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" root@${server_ip}:.; then
+        break
+    fi
+    echo "Timedout, trying again in 30 seconds"
+    sleep 30
+done
+
+echo "Copying INFRA_VERSION"
+$scp22 "${SCRIPT_DIR}/../setup/INFRA_VERSION" root@${server_ip}:.
+
+echo "Copying box source"
+cd "${SOURCE_DIR}"
+git archive --format=tar HEAD | $ssh22 "root@${server_ip}" "cat - > /tmp/box.tar.gz"
+
+echo "Executing init script"
+if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then
+    echo "Init script failed"
+    exit 1
+fi
+
+echo "Shutting down server with id : ${server_id}"
+$ssh202 "root@${server_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail
+
+# wait 10 secs for actual shutdown
+echo "Waiting for 10 seconds for server to shutdown"
+sleep 30
+
+echo "Powering off server"
+if ! $vps power_off "${server_id}"; then
+    echo "Could not power off server"
+    exit 1
+fi
+
+snapshot_name="box-${deploy_env}-${pretty_revision}-${now}"
+echo "Snapshotting as ${snapshot_name}"
+if ! image_id=$($vps snapshot "${server_id}" "${snapshot_name}"); then
+    echo "Could not snapshot and get image id"
+    exit 1
+fi
+
+if [[ "${destroy_server}" == "yes" ]]; then
+    echo "Destroying server"
+    if ! $vps destroy "${server_id}"; then
+        echo "Could not destroy server"
+        exit 1
+    fi
+else
+    echo "Skipping server destroy"
+fi
+
+echo "Transferring image ${image_id} to other regions"
+$vps transfer_image_to_all_regions "${image_id}"
+
+echo "Done."

baseimage/digitalocean.sh

@@ -0,0 +1,240 @@
+#!/bin/bash
+
+if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then
+    echo "Script requires DIGITAL_OCEAN_TOKEN env to be set"
+    exit 1
+fi
+
+if [[ -z "${JSON}" ]]; then
+    echo "Script requires JSON env to be set to path of JSON binary"
+    exit 1
+fi
+
+readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+
+function debug() {
+    echo "$@" >&2
+}
+
+function get_ssh_key_id() {
+    id=$($CURL "https://api.digitalocean.com/v2/account/keys" \
+        | $JSON ssh_keys \
+        | $JSON -c "this.name === \"$1\"" \
+        | $JSON 0.id)
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function create_droplet() {
+    local ssh_key_id="$1"
+    local box_name="$2"
+
+    local image_region="sfo1"
+    local ubuntu_image_slug="ubuntu-15-10-x64"
+    local box_size="512mb"
+
+    local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
+
+    id=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id)
+    [[ -z "$id" ]] && exit 1
+    echo "$id"
+}
+
+function get_droplet_ip() {
+    local droplet_id="$1"
+    ip=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address")
+    [[ -z "$ip" ]] && exit 1
+    echo "$ip"
+}
+
+function get_droplet_id() {
+    local droplet_name="$1"
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    [[ -z "$id" ]] && exit 1
+    echo "$id"  
+}
+
+function power_off_droplet() {
+    local droplet_id="$1"
+    local data='{"type":"power_off"}'
+    local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions")
+    local event_id=`echo "${response}" | $JSON action.id`
+
+    if [[ -z "${event_id}" ]]; then
+        debug "Got no event id, assuming already powered off."
+        debug "Response: ${response}"
+        return
+    fi
+
+    debug "Powered off droplet. Event id: ${event_id}"
+    debug -n "Waiting for droplet to power off"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function power_on_droplet() {
+    local droplet_id="$1"
+    local data='{"type":"power_on"}'
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
+
+    debug "Powered on droplet. Event id: ${event_id}"
+
+    if [[ -z "${event_id}" ]]; then
+        debug "Got no event id, assuming already powered on"
+        return
+    fi
+
+    debug -n "Waiting for droplet to power on"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function get_image_id() {
+    local snapshot_name="$1"
+    local image_id=""
+
+    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
+
+    if [[ -n "${image_id}" ]]; then
+        echo "${image_id}"
+    fi
+}
+
+function snapshot_droplet() {
+    local droplet_id="$1"
+    local snapshot_name="$2"
+    local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}"
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id`
+
+    debug "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}"
+    debug -n "Waiting for snapshot to complete"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+
+    get_image_id "${snapshot_name}"
+}
+
+function destroy_droplet() {
+    local droplet_id="$1"
+    # TODO: check for 204 status
+    $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}"
+    debug "Droplet destroyed"
+    debug ""
+}
+
+function transfer_image() {
+    local image_id="$1"
+    local region_slug="$2"
+    local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}"
+    local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id`
+    echo "${event_id}"
+}
+
+function wait_for_image_event() {
+    local image_id="$1"
+    local event_id="$2"
+
+    debug -n "Waiting for ${event_id}"
+
+    while true; do
+        local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status`
+        if [[ "${event_status}" == "completed" ]]; then
+            break
+        fi
+        debug -n "."
+        sleep 10
+    done
+    debug ""
+}
+
+function transfer_image_to_all_regions() {
+    local image_id="$1"
+
+    xfer_events=()
+    image_regions=(ams3) ## sfo1 is where the image is created
+    for image_region in ${image_regions[@]}; do
+        xfer_event=$(transfer_image ${image_id} ${image_region})
+        echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"
+        xfer_events+=("${xfer_event}")
+        sleep 1
+    done
+
+    echo "Image transfer initiated, but they will take some time to get transferred."
+
+    for xfer_event in ${xfer_events[@]}; do
+        $vps wait_for_image_event "${image_id}" "${xfer_event}"
+    done
+}
+
+if [[ $# -lt 1 ]]; then
+    debug "<command> <params...>"
+    exit 1
+fi
+
+case $1 in
+get_ssh_key_id)
+    get_ssh_key_id "${@:2}"
+    ;;
+
+create)
+    create_droplet "${@:2}"
+    ;;
+
+get_id)
+    get_droplet_id "${@:2}"
+    ;;
+
+get_ip)
+    get_droplet_ip "${@:2}"
+    ;;
+
+power_on)
+    power_on_droplet "${@:2}"
+    ;;
+
+power_off)
+    power_off_droplet "${@:2}"
+    ;;
+
+snapshot)
+    snapshot_droplet "${@:2}"
+    ;;
+
+destroy)
+    destroy_droplet "${@:2}"
+    ;;
+
+transfer_image_to_all_regions)
+    transfer_image_to_all_regions "${@:2}"
+    ;;
+
+*)
+    echo "Unknown command $1"
+    exit 1
+esac

baseimage/initializeBaseUbuntuImage.sh

@@ -0,0 +1,326 @@
+#!/bin/bash
+
+set -euv -o pipefail
+
+readonly USER=yellowtent
+readonly USER_HOME="/home/${USER}"
+readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
+readonly INSTALLER_REVISION="$1"
+readonly SELFHOSTED=$(( $# > 1 ? 1 : 0 ))
+readonly USER_DATA_FILE="/root/user_data.img"
+readonly USER_DATA_DIR="/home/yellowtent/data"
+
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+function die {
+    echo $1
+    exit 1
+}
+
+[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
+
+if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
+    source "${SOURCE_DIR}/INFRA_VERSION"
+else
+    echo "No INFRA_VERSION found, skip pulling docker images"
+fi
+
+if [ ${SELFHOSTED} == 0 ]; then
+    echo "!! Initializing Ubuntu image for CaaS"
+else
+    echo "!! Initializing Ubuntu image for Selfhosting"
+fi
+
+echo "==== Create User ${USER} ===="
+if ! id "${USER}"; then
+    useradd "${USER}" -m
+fi
+
+echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ==="
+
+echo "=== Prepare installer source ==="
+rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}"
+rm -rf /tmp/box && mkdir -p /tmp/box
+tar xvf /tmp/box.tar.gz -C /tmp/box && rm /tmp/box.tar.gz
+cp -rf /tmp/box/installer/* "${INSTALLER_SOURCE_DIR}"
+echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION"
+
+export DEBIAN_FRONTEND=noninteractive
+
+echo "=== Upgrade ==="
+apt-get update
+apt-get dist-upgrade -y
+apt-get install -y curl
+
+# Setup firewall before everything. docker creates it's own chain and the -X below will remove it
+# Do NOT use iptables-persistent because it's startup ordering conflicts with docker
+echo "=== Setting up firewall ==="
+# clear tables and set default policy
+iptables -F # flush all chains
+iptables -X # delete all chains
+# default policy for filter table
+iptables -P INPUT DROP
+iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject
+iptables -P OUTPUT ACCEPT
+
+# NOTE: keep these in sync with src/apps.js validatePortBindings
+# allow ssh, http, https, ping, dns
+iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
+if [ ${SELFHOSTED} == 0 ]; then
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
+else
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
+fi
+iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
+iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
+iptables -A INPUT -p udp --sport 53 -j ACCEPT
+iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+
+# loopback
+iptables -A INPUT -i lo -j ACCEPT
+iptables -A OUTPUT -o lo -j ACCEPT
+
+# prevent DoS
+# iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT
+
+# log dropped incoming. keep this at the end of all the rules
+iptables -N LOGGING # new chain
+iptables -A INPUT -j LOGGING # last rule in INPUT chain
+iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
+iptables -A LOGGING -j DROP
+
+echo "==== Install btrfs tools ==="
+apt-get -y install btrfs-tools
+
+echo "==== Install docker ===="
+# install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
+curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
+apt-get -y install aufs-tools
+chmod +x /usr/bin/docker
+groupadd docker
+cat > /etc/systemd/system/docker.socket <<EOF
+[Unit]
+Description=Docker Socket for the API
+PartOf=docker.service
+
+[Socket]
+ListenStream=/var/run/docker.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target
+EOF
+cat > /etc/systemd/system/docker.service <<EOF
+[Unit]
+Description=Docker Application Container Engine
+After=network.target docker.socket
+Requires=docker.socket
+
+[Service]
+ExecStart=/usr/bin/docker daemon -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+echo "=== Setup btrfs data ==="
+truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
+mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
+mkdir -p "${USER_DATA_DIR}"
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
+
+systemctl daemon-reload
+systemctl enable docker
+systemctl start docker
+
+# give docker sometime to start up and create iptables rules
+# those rules come in after docker has started, and we want to wait for them to be sure iptables-save has all of them
+sleep 10
+
+# Disable forwarding to metadata route from containers
+iptables -I FORWARD -d 169.254.169.254 -j DROP
+
+# ubuntu will restore iptables from this file automatically. this is here so that docker's chain is saved to this file
+mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4
+
+echo "=== Enable memory accounting =="
+sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
+
+# now add the user to the docker group
+usermod "${USER}" -a -G docker
+
+if [ -z $(echo "${INFRA_VERSION}") ]; then
+    echo "Skip pulling base docker images"
+else
+    echo "=== Pulling base docker images ==="
+    docker pull "${BASE_IMAGE}"
+
+    echo "=== Pulling mysql addon image ==="
+    docker pull "${MYSQL_IMAGE}"
+
+    echo "=== Pulling postgresql addon image ==="
+    docker pull "${POSTGRESQL_IMAGE}"
+
+    echo "=== Pulling redis addon image ==="
+    docker pull "${REDIS_IMAGE}"
+
+    echo "=== Pulling mongodb addon image ==="
+    docker pull "${MONGODB_IMAGE}"
+
+    echo "=== Pulling graphite docker images ==="
+    docker pull "${GRAPHITE_IMAGE}"
+
+    echo "=== Pulling mail relay ==="
+    docker pull "${MAIL_IMAGE}"
+fi
+
+echo "==== Install nginx ===="
+apt-get -y install nginx-full
+[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
+
+echo "==== Install build-essential ===="
+apt-get -y install build-essential rcconf
+
+echo "==== Install mysql ===="
+debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
+debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
+apt-get -y install mysql-server
+[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
+
+echo "==== Install pwgen ===="
+apt-get -y install pwgen
+
+echo "==== Install collectd ==="
+if ! apt-get install -y collectd collectd-utils; then
+    # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
+    echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
+    sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
+fi
+update-rc.d -f collectd remove
+
+# this simply makes it explicit that we run logrotate via cron. it's already part of base ubuntu
+echo "==== Install logrotate ==="
+apt-get install -y cron logrotate
+systemctl enable cron
+
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
+
+echo "=== Rebuilding npm packages ==="
+cd "${INSTALLER_SOURCE_DIR}" && npm install --production
+chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
+
+echo "==== Install installer systemd script ===="
+provisionEnv="PROVISION=digitalocean"
+if [ ${SELFHOSTED} == 1 ]; then
+    provisionEnv="PROVISION=local"
+fi
+
+cat > /etc/systemd/system/cloudron-installer.service <<EOF
+[Unit]
+Description=Cloudron Installer
+; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
+BindsTo=systemd-journald.service
+
+[Service]
+Type=idle
+ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
+Environment="DEBUG=installer*,connect-lastmile" ${provisionEnv}
+; kill any child (installer.sh) as well
+KillMode=control-group
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Restore iptables before docker
+echo "==== Install iptables-restore systemd script ===="
+cat > /etc/systemd/system/iptables-restore.service <<EOF
+[Unit]
+Description=IPTables Restore
+Before=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/sbin/iptables-restore /etc/iptables/rules.v4
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Allocate swap files
+# https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
+echo "==== Install box-setup systemd script ===="
+cat > /etc/systemd/system/box-setup.service <<EOF
+[Unit]
+Description=Box Setup
+Before=docker.service collectd.service mysql.service
+After=do-resize.service
+
+[Service]
+Type=oneshot
+ExecStart="${INSTALLER_SOURCE_DIR}/systemd/box-setup.sh"
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+systemctl daemon-reload
+systemctl enable cloudron-installer
+systemctl enable iptables-restore
+systemctl enable box-setup
+
+# Configure systemd
+sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
+    -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
+    -i /etc/systemd/journald.conf
+
+# When rotating logs, systemd kills journald too soon sometimes
+# See https://github.com/systemd/systemd/issues/1353 (this is upstream default)
+sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
+    -i /lib/systemd/system/systemd-journald.service
+
+sync
+
+# Configure time
+sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
+timedatectl set-ntp 1
+timedatectl set-timezone UTC
+
+# Give user access to system logs
+apt-get -y install acl
+usermod -a -G systemd-journal ${USER}
+mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
+chown root:systemd-journal /var/log/journal
+systemctl restart systemd-journald
+setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
+
+if [ ${SELFHOSTED} == 0 ]; then
+    echo "==== Install ssh ==="
+    apt-get -y install openssh-server
+    # https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
+    sed -e 's/^#\?Port .*/Port 202/g' \
+        -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
+        -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
+        -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
+        -i /etc/ssh/sshd_config
+
+    # required so we can connect to this machine since port 22 is blocked by iptables by now
+    systemctl reload sshd
+fi

box.js

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+
+'use strict';
+
+require('supererror')({ splatchError: true });
+
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
+var appHealthMonitor = require('./src/apphealthmonitor.js'),
+    async = require('async'),
+    config = require('./src/config.js'),
+    ldap = require('./src/ldap.js'),
+    oauthproxy = require('./src/oauthproxy.js'),
+    server = require('./src/server.js'),
+    simpleauth = require('./src/simpleauth.js');
+
+console.log();
+console.log('==========================================');
+console.log(' Cloudron will use the following settings ');
+console.log('==========================================');
+console.log();
+console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
+console.log(' Version:                        ', config.version());
+console.log(' Admin Origin:                   ', config.adminOrigin());
+console.log(' Appstore API server origin:     ', config.apiServerOrigin());
+console.log(' Appstore Web server origin:     ', config.webServerOrigin());
+console.log();
+console.log('==========================================');
+console.log();
+
+async.series([
+    server.start,
+    ldap.start,
+    simpleauth.start,
+    appHealthMonitor.start,
+    oauthproxy.start
+], function (error) {
+    if (error) {
+        console.error('Error starting server', error);
+        process.exit(1);
+    }
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    simpleauth.stop(NOOP_CALLBACK);
+    oauthproxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});

crashnotifier.js

@@ -2,20 +2,12 @@
 
 'use strict';
 
-// WARNING This is a supervisor eventlistener!
-//         The communication happens via stdin/stdout
-//         !! No console.log() allowed
-//         !! Do not set DEBUG
-
 var assert = require('assert'),
     mailer = require('./src/mailer.js'),
     safe = require('safetydance'),
-    supervisor = require('supervisord-eventlistener'),
     path = require('path'),
     util = require('util');
 
-var gLastNotifyTime = {};
-var gCooldownTime = 1000 * 60  * 5; // 5 min
 var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
 
 function collectLogs(program, callback) {
@@ -26,28 +18,25 @@ function collectLogs(program, callback) {
     callback(null, logs);
 }
 
-supervisor.on('PROCESS_STATE_EXITED', function (headers, data) {
+function sendCrashNotification(processName) {
-    if (data.expected === '1') return console.error('Normal app %s exit', data.processname);
+    collectLogs(processName, function (error, result) {
-
-    console.error('%s exited unexpectedly', data.processname);
-
-    collectLogs(data.processname, function (error, result) {
         if (error) {
             console.error('Failed to collect logs.', error);
             result = util.format('Failed to collect logs.', error);
         }
 
-        if (!gLastNotifyTime[data.processname] || gLastNotifyTime[data.processname] < Date.now() - gCooldownTime) {
+        console.log('Sending crash notification email for', processName);
-            console.error('Send mail.');
+        mailer.sendCrashNotification(processName, result);
-            mailer.sendCrashNotification(data.processname, result);
-            gLastNotifyTime[data.processname] = Date.now();
-        } else {
-            console.error('Do not send mail, already sent one recently.');
-        }
     });
-});
+}
 
-mailer.initialize(function () {
+function main() {
-    supervisor.listen(process.stdin, process.stdout);
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-    console.error('Crashnotifier listening...');
+
-});
+    var processName = process.argv[2];
+    console.log('Started crash notifier for', processName);
+
+    sendCrashNotification(processName);
+}
+
+main();

gulpfile.js

@@ -10,7 +10,7 @@ var ejs = require('gulp-ejs'),
     serve = require('gulp-serve'),
     sass = require('gulp-sass'),
     sourcemaps = require('gulp-sourcemaps'),
-    minifyCSS = require('gulp-minify-css'),
+    cssnano = require('gulp-cssnano'),
     autoprefixer = require('gulp-autoprefixer'),
     argv = require('yargs').argv;
 
@@ -39,7 +39,7 @@ gulp.task('3rdparty', function () {
 // JavaScript
 // --------------
 
-gulp.task('js', ['js-index', 'js-setup', 'js-update', 'js-error'], function () {});
+gulp.task('js', ['js-index', 'js-setup', 'js-update'], function () {});
 
 var oauth = {
     clientId: argv.clientId || 'cid-webadmin',
@@ -80,14 +80,6 @@ gulp.task('js-setup', function () {
         .pipe(gulp.dest('webadmin/dist/js'));
 });
 
-gulp.task('js-error', function () {
-    gulp.src(['webadmin/src/js/error.js'])
-        .pipe(sourcemaps.init())
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'));
-});
-
 gulp.task('js-update', function () {
     gulp.src(['webadmin/src/js/update.js'])
         .pipe(sourcemaps.init())
@@ -102,7 +94,7 @@ gulp.task('js-update', function () {
 // HTML
 // --------------
 
-gulp.task('html', ['html-views', 'html-update'], function () {
+gulp.task('html', ['html-views', 'html-update', 'html-templates'], function () {
     return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
 });
 
@@ -114,6 +106,10 @@ gulp.task('html-views', function () {
     return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
 });
 
+gulp.task('html-templates', function () {
+    return gulp.src('webadmin/src/templates/**/*.html').pipe(gulp.dest('webadmin/dist/templates'));
+});
+
 // --------------
 // CSS
 // --------------
@@ -123,7 +119,7 @@ gulp.task('css', function () {
         .pipe(sourcemaps.init())
         .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
         .pipe(autoprefixer())
-        .pipe(minifyCSS())
+        .pipe(cssnano())
         .pipe(sourcemaps.write())
         .pipe(gulp.dest('webadmin/dist'))
         .pipe(gulp.dest('setup/splash/website'));
@@ -143,8 +139,8 @@ gulp.task('watch', ['default'], function () {
     gulp.watch(['webadmin/src/img/*'], ['images']);
     gulp.watch(['webadmin/src/**/*.html'], ['html']);
     gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
+    gulp.watch(['webadmin/src/templates/*.html'], ['html-templates']);
     gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
-    gulp.watch(['webadmin/src/js/error.js'], ['js-error']);
     gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
     gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
     gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);

installer.sh.ejs

@@ -0,0 +1,164 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+echo ""
+echo "======== Cloudron Installer ========"
+echo ""
+
+if [ $# -lt 4 ]; then
+    echo "Usage: ./installer.sh <fqdn> <aws key id> <aws key secret> <bucket> <provider> <revision>"
+    exit 1
+fi
+
+# commandline arguments
+readonly fqdn="${1}"
+readonly aws_access_key_id="${2}"
+readonly aws_access_key_secret="${3}"
+readonly aws_backup_bucket="${4}"
+readonly provider="${5}"
+readonly revision="${6}"
+
+# environment specific urls
+<% if (env === 'prod') { %>
+readonly api_server_origin="https://api.cloudron.io"
+readonly web_server_origin="https://cloudron.io"
+<% } else { %>
+readonly api_server_origin="https://api.<%= env %>.cloudron.io"
+readonly web_server_origin="https://<%= env %>.cloudron.io"
+<% } %>
+readonly release_bucket_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases"
+readonly versions_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases/versions.json"
+readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
+
+# runtime consts
+readonly installer_code_file="/tmp/box.tar.gz"
+readonly installer_tmp_dir="/tmp/box"
+readonly cert_folder="/tmp/certificates"
+
+# check for fqdn in /ets/hosts
+echo "[INFO] checking for hostname entry"
+readonly hostentry_found=$(grep "${fqdn}" /etc/hosts || true)
+if [[ -z $hostentry_found ]]; then
+    echo "[WARNING] No entry for ${fqdn} found in /etc/hosts"
+    echo "Adding an entry ..."
+
+    cat >> /etc/hosts <<EOF
+
+# The following line was added by the Cloudron installer script
+127.0.1.1 ${fqdn} ${fqdn}
+EOF
+else
+    echo "Valid hostname entry found in /etc/hosts"
+fi
+echo ""
+
+echo "[INFO] ensure minimal dependencies ..."
+apt-get update
+apt-get install -y curl
+echo ""
+
+echo "[INFO] Generating certificates ..."
+rm -rf "${cert_folder}"
+mkdir -p "${cert_folder}"
+
+cat > "${cert_folder}/CONFIG" <<EOF
+[ req ]
+default_bits           = 1024
+default_keyfile        = keyfile.pem
+distinguished_name     = req_distinguished_name
+prompt                 = no
+req_extensions         = v3_req
+
+[ req_distinguished_name ]
+C                      = DE
+ST                     = Berlin
+L                      = Berlin
+O                      = Cloudron UG
+OU                     = Cloudron
+CN                     = ${fqdn}
+emailAddress           = cert@cloudron.io
+
+[ v3_req ]
+# Extensions to add to a certificate request
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = ${fqdn}
+DNS.2 = *.${fqdn}
+EOF
+
+# generate cert files
+openssl genrsa 2048 > "${cert_folder}/host.key"
+openssl req -new -out "${cert_folder}/host.csr" -key "${cert_folder}/host.key" -config "${cert_folder}/CONFIG"
+openssl x509 -req -days 3650 -in "${cert_folder}/host.csr" -signkey "${cert_folder}/host.key" -out "${cert_folder}/host.cert" -extensions v3_req -extfile "${cert_folder}/CONFIG"
+
+# make them json compatible, by collapsing to one line
+tls_cert=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.cert")
+tls_key=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.key")
+echo ""
+
+echo "[INFO] Fetching installer code ..."
+curl "${installer_code_url}" -o "${installer_code_file}"
+echo ""
+
+echo "[INFO] Extracting installer code to ${installer_tmp_dir} ..."
+rm -rf "${installer_tmp_dir}" && mkdir -p "${installer_tmp_dir}"
+tar xvf "${installer_code_file}" -C "${installer_tmp_dir}"
+echo ""
+
+echo "Creating initial provisioning config ..."
+cat > /root/provision.json <<EOF
+{
+    "sourceTarballUrl": "",
+    "data": {
+        "apiServerOrigin": "${api_server_origin}",
+        "webServerOrigin": "${web_server_origin}",
+        "fqdn": "${fqdn}",
+        "token": "",
+        "isCustomDomain": true,
+        "boxVersionsUrl": "${versions_url}",
+        "version": "",
+        "tlsCert": "${tls_cert}",
+        "tlsKey": "${tls_key}",
+        "provider": "${provider}",
+        "backupConfig": {
+            "provider": "s3",
+            "accessKeyId": "${aws_access_key_id}",
+            "secretAccessKey": "${aws_access_key_secret}",
+            "bucket": "${aws_backup_bucket}",
+            "prefix": "backups"
+        },
+        "dnsConfig": {
+            "provider": "route53",
+            "accessKeyId": "${aws_access_key_id}",
+            "secretAccessKey": "${aws_access_key_secret}"
+        },
+        "tlsConfig": {
+            "provider": "letsencrypt-<%= env %>"
+        }
+    }
+}
+EOF
+
+echo "[INFO] Running Ubuntu initializing script ..."
+/bin/bash "${installer_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${revision}" selfhosting
+echo ""
+
+echo "[INFO] Reloading systemd daemon ..."
+systemctl daemon-reload
+echo ""
+
+echo "[INFO] Restart docker ..."
+systemctl restart docker
+echo ""
+
+echo "[FINISHED] Now starting Cloudron init jobs ..."
+systemctl start box-setup
+
+# TODO this is only for convenience we should probably just let the user do a restart
+sleep 5 && sync
+systemctl start cloudron-installer
+journalctl -u cloudron-installer.service -f

installer/npm-shrinkwrap.json

@@ -0,0 +1,516 @@
+{
+  "name": "installer",
+  "version": "0.0.1",
+  "dependencies": {
+    "async": {
+      "version": "1.5.0",
+      "from": "async@>=1.5.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz"
+    },
+    "body-parser": {
+      "version": "1.14.1",
+      "from": "body-parser@>=1.12.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
+      "dependencies": {
+        "bytes": {
+          "version": "2.1.0",
+          "from": "bytes@2.1.0",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz"
+        },
+        "content-type": {
+          "version": "1.0.1",
+          "from": "content-type@>=1.0.1 <1.1.0",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
+        },
+        "depd": {
+          "version": "1.1.0",
+          "from": "depd@>=1.1.0 <1.2.0",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
+        },
+        "http-errors": {
+          "version": "1.3.1",
+          "from": "http-errors@>=1.3.1 <1.4.0",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+          "dependencies": {
+            "inherits": {
+              "version": "2.0.1",
+              "from": "inherits@>=2.0.1 <2.1.0",
+              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+            },
+            "statuses": {
+              "version": "1.2.1",
+              "from": "statuses@>=1.0.0 <2.0.0",
+              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+            }
+          }
+        },
+        "iconv-lite": {
+          "version": "0.4.12",
+          "from": "iconv-lite@0.4.12",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "on-finished@>=2.3.0 <2.4.0",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "ee-first@1.1.1",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "qs": {
+          "version": "5.1.0",
+          "from": "qs@5.1.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz"
+        },
+        "raw-body": {
+          "version": "2.1.4",
+          "from": "raw-body@>=2.1.4 <2.2.0",
+          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
+          "dependencies": {
+            "unpipe": {
+              "version": "1.0.0",
+              "from": "unpipe@1.0.0",
+              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+            }
+          }
+        },
+        "type-is": {
+          "version": "1.6.9",
+          "from": "type-is@>=1.6.9 <1.7.0",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "dependencies": {
+            "media-typer": {
+              "version": "0.3.0",
+              "from": "media-typer@0.3.0",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "mime-types@>=2.1.7 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "mime-db@>=1.19.0 <1.20.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "connect-lastmile": {
+      "version": "0.0.13",
+      "from": "connect-lastmile@0.0.13",
+      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
+      "dependencies": {
+        "debug": {
+          "version": "2.1.3",
+          "from": "debug@>=2.1.0 <2.2.0",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
+          "dependencies": {
+            "ms": {
+              "version": "0.7.0",
+              "from": "ms@0.7.0",
+              "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
+            }
+          }
+        }
+      }
+    },
+    "debug": {
+      "version": "2.2.0",
+      "from": "debug@>=2.1.1 <3.0.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
+      "dependencies": {
+        "ms": {
+          "version": "0.7.1",
+          "from": "ms@0.7.1",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+        }
+      }
+    },
+    "express": {
+      "version": "4.13.3",
+      "from": "express@>=4.11.2 <5.0.0",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
+      "dependencies": {
+        "accepts": {
+          "version": "1.2.13",
+          "from": "accepts@>=1.2.12 <1.3.0",
+          "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
+          "dependencies": {
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "mime-types@>=2.1.6 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "mime-db@>=1.19.0 <1.20.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            },
+            "negotiator": {
+              "version": "0.5.3",
+              "from": "negotiator@0.5.3",
+              "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
+            }
+          }
+        },
+        "array-flatten": {
+          "version": "1.1.1",
+          "from": "array-flatten@1.1.1",
+          "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
+        },
+        "content-disposition": {
+          "version": "0.5.0",
+          "from": "content-disposition@0.5.0",
+          "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz"
+        },
+        "content-type": {
+          "version": "1.0.1",
+          "from": "content-type@>=1.0.1 <1.1.0",
+          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
+        },
+        "cookie": {
+          "version": "0.1.3",
+          "from": "cookie@0.1.3",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz"
+        },
+        "cookie-signature": {
+          "version": "1.0.6",
+          "from": "cookie-signature@1.0.6",
+          "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
+        },
+        "depd": {
+          "version": "1.0.1",
+          "from": "depd@>=1.0.1 <1.1.0",
+          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+        },
+        "escape-html": {
+          "version": "1.0.2",
+          "from": "escape-html@1.0.2",
+          "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz"
+        },
+        "etag": {
+          "version": "1.7.0",
+          "from": "etag@>=1.7.0 <1.8.0",
+          "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
+        },
+        "finalhandler": {
+          "version": "0.4.0",
+          "from": "finalhandler@0.4.0",
+          "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
+          "dependencies": {
+            "unpipe": {
+              "version": "1.0.0",
+              "from": "unpipe@>=1.0.0 <1.1.0",
+              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
+            }
+          }
+        },
+        "fresh": {
+          "version": "0.3.0",
+          "from": "fresh@0.3.0",
+          "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
+        },
+        "merge-descriptors": {
+          "version": "1.0.0",
+          "from": "merge-descriptors@1.0.0",
+          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
+        },
+        "methods": {
+          "version": "1.1.1",
+          "from": "methods@>=1.1.1 <1.2.0",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "on-finished@>=2.3.0 <2.4.0",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "ee-first@1.1.1",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "parseurl": {
+          "version": "1.3.0",
+          "from": "parseurl@>=1.3.0 <1.4.0",
+          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
+        },
+        "path-to-regexp": {
+          "version": "0.1.7",
+          "from": "path-to-regexp@0.1.7",
+          "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
+        },
+        "proxy-addr": {
+          "version": "1.0.8",
+          "from": "proxy-addr@>=1.0.8 <1.1.0",
+          "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
+          "dependencies": {
+            "forwarded": {
+              "version": "0.1.0",
+              "from": "forwarded@>=0.1.0 <0.2.0",
+              "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz"
+            },
+            "ipaddr.js": {
+              "version": "1.0.1",
+              "from": "ipaddr.js@1.0.1",
+              "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz"
+            }
+          }
+        },
+        "qs": {
+          "version": "4.0.0",
+          "from": "qs@4.0.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
+        },
+        "range-parser": {
+          "version": "1.0.3",
+          "from": "range-parser@>=1.0.2 <1.1.0",
+          "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
+        },
+        "send": {
+          "version": "0.13.0",
+          "from": "send@0.13.0",
+          "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
+          "dependencies": {
+            "destroy": {
+              "version": "1.0.3",
+              "from": "destroy@1.0.3",
+              "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz"
+            },
+            "http-errors": {
+              "version": "1.3.1",
+              "from": "http-errors@>=1.3.1 <1.4.0",
+              "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
+              "dependencies": {
+                "inherits": {
+                  "version": "2.0.1",
+                  "from": "inherits@>=2.0.1 <2.1.0",
+                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+                }
+              }
+            },
+            "mime": {
+              "version": "1.3.4",
+              "from": "mime@1.3.4",
+              "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
+            },
+            "ms": {
+              "version": "0.7.1",
+              "from": "ms@0.7.1",
+              "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
+            },
+            "statuses": {
+              "version": "1.2.1",
+              "from": "statuses@>=1.2.1 <1.3.0",
+              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
+            }
+          }
+        },
+        "serve-static": {
+          "version": "1.10.0",
+          "from": "serve-static@>=1.10.0 <1.11.0",
+          "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
+        },
+        "type-is": {
+          "version": "1.6.9",
+          "from": "type-is@>=1.6.9 <1.7.0",
+          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
+          "dependencies": {
+            "media-typer": {
+              "version": "0.3.0",
+              "from": "media-typer@0.3.0",
+              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.7",
+              "from": "mime-types@>=2.1.6 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.19.0",
+                  "from": "mime-db@>=1.19.0 <1.20.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
+                }
+              }
+            }
+          }
+        },
+        "utils-merge": {
+          "version": "1.0.0",
+          "from": "utils-merge@1.0.0",
+          "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
+        },
+        "vary": {
+          "version": "1.0.1",
+          "from": "vary@>=1.0.1 <1.1.0",
+          "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
+        }
+      }
+    },
+    "json": {
+      "version": "9.0.3",
+      "from": "json@>=9.0.3 <10.0.0",
+      "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz"
+    },
+    "morgan": {
+      "version": "1.6.1",
+      "from": "morgan@>=1.5.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
+      "dependencies": {
+        "basic-auth": {
+          "version": "1.0.3",
+          "from": "basic-auth@>=1.0.3 <1.1.0",
+          "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz"
+        },
+        "depd": {
+          "version": "1.0.1",
+          "from": "depd@>=1.0.1 <1.1.0",
+          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
+        },
+        "on-finished": {
+          "version": "2.3.0",
+          "from": "on-finished@>=2.3.0 <2.4.0",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
+          "dependencies": {
+            "ee-first": {
+              "version": "1.1.1",
+              "from": "ee-first@1.1.1",
+              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
+            }
+          }
+        },
+        "on-headers": {
+          "version": "1.0.1",
+          "from": "on-headers@>=1.0.0 <1.1.0",
+          "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz"
+        }
+      }
+    },
+    "proxy-middleware": {
+      "version": "0.15.0",
+      "from": "proxy-middleware@>=0.15.0 <0.16.0",
+      "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz"
+    },
+    "safetydance": {
+      "version": "0.0.19",
+      "from": "safetydance@0.0.19",
+      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
+    },
+    "semver": {
+      "version": "5.1.0",
+      "from": "semver@>=5.1.0 <6.0.0",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
+    },
+    "superagent": {
+      "version": "0.21.0",
+      "from": "superagent@>=0.21.0 <0.22.0",
+      "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
+      "dependencies": {
+        "component-emitter": {
+          "version": "1.1.2",
+          "from": "component-emitter@1.1.2",
+          "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz"
+        },
+        "cookiejar": {
+          "version": "2.0.1",
+          "from": "cookiejar@2.0.1",
+          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz"
+        },
+        "extend": {
+          "version": "1.2.1",
+          "from": "extend@>=1.2.1 <1.3.0",
+          "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz"
+        },
+        "form-data": {
+          "version": "0.1.3",
+          "from": "form-data@0.1.3",
+          "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
+          "dependencies": {
+            "async": {
+              "version": "0.9.2",
+              "from": "async@>=0.9.0 <0.10.0",
+              "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
+            },
+            "combined-stream": {
+              "version": "0.0.7",
+              "from": "combined-stream@>=0.0.4 <0.1.0",
+              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
+              "dependencies": {
+                "delayed-stream": {
+                  "version": "0.0.5",
+                  "from": "delayed-stream@0.0.5",
+                  "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
+                }
+              }
+            }
+          }
+        },
+        "formidable": {
+          "version": "1.0.14",
+          "from": "formidable@1.0.14",
+          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz"
+        },
+        "methods": {
+          "version": "1.0.1",
+          "from": "methods@1.0.1",
+          "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz"
+        },
+        "mime": {
+          "version": "1.2.11",
+          "from": "mime@1.2.11",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
+        },
+        "qs": {
+          "version": "1.2.0",
+          "from": "qs@1.2.0",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz"
+        },
+        "readable-stream": {
+          "version": "1.0.27-1",
+          "from": "readable-stream@1.0.27-1",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
+          "dependencies": {
+            "core-util-is": {
+              "version": "1.0.1",
+              "from": "core-util-is@>=1.0.0 <1.1.0",
+              "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
+            },
+            "inherits": {
+              "version": "2.0.1",
+              "from": "inherits@>=2.0.1 <2.1.0",
+              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
+            },
+            "isarray": {
+              "version": "0.0.1",
+              "from": "isarray@0.0.1",
+              "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
+            },
+            "string_decoder": {
+              "version": "0.10.31",
+              "from": "string_decoder@>=0.10.0 <0.11.0",
+              "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
+            }
+          }
+        },
+        "reduce-component": {
+          "version": "1.0.1",
+          "from": "reduce-component@1.0.1",
+          "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
+        }
+      }
+    }
+  }
+}

installer/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "installer",
+  "description": "Cloudron Installer",
+  "version": "0.0.1",
+  "private": "true",
+  "author": {
+    "name": "Cloudron authors"
+  },
+  "repository": {
+    "type": "git"
+  },
+  "engines": [
+    "node >=4.0.0 <=4.1.1"
+  ],
+  "dependencies": {
+    "async": "^1.5.0",
+    "body-parser": "^1.12.0",
+    "connect-lastmile": "0.0.13",
+    "debug": "^2.1.1",
+    "express": "^4.11.2",
+    "json": "^9.0.3",
+    "morgan": "^1.5.1",
+    "proxy-middleware": "^0.15.0",
+    "safetydance": "0.0.19",
+    "semver": "^5.1.0",
+    "superagent": "^0.21.0"
+  },
+  "devDependencies": {
+    "colors": "^1.1.2",
+    "commander": "^2.8.1",
+    "expect.js": "^0.3.1",
+    "istanbul": "^0.3.5",
+    "lodash": "^3.2.0",
+    "mocha": "^2.1.0",
+    "nock": "^0.59.1",
+    "sleep": "^3.0.0",
+    "superagent-sync": "^0.2.0",
+    "supererror": "^0.7.0",
+    "yesno": "0.0.1"
+  },
+  "scripts": {
+    "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test",
+    "precommit": "/bin/true",
+    "prepush": "npm test",
+    "postmerge": "/bin/true"
+  }
+}

installer/src/installer.js

@@ -0,0 +1,112 @@
+/* jslint node: true */
+
+'use strict';
+
+var assert = require('assert'),
+    child_process = require('child_process'),
+    debug = require('debug')('installer:installer'),
+    path = require('path'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    superagent = require('superagent'),
+    util = require('util');
+
+exports = module.exports = {
+    InstallerError: InstallerError,
+
+    provision: provision,
+
+    _ensureVersion: ensureVersion
+};
+
+var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'),
+    SUDO = '/usr/bin/sudo';
+
+function InstallerError(reason, info) {
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info);
+}
+util.inherits(InstallerError, Error);
+InstallerError.INTERNAL_ERROR = 1;
+InstallerError.ALREADY_PROVISIONED = 2;
+
+// system until file has KillMode=control-group to bring down child processes
+function spawn(tag, cmd, args, callback) {
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof cmd, 'string');
+    assert(util.isArray(args));
+    assert.strictEqual(typeof callback, 'function');
+
+    var cp = child_process.spawn(cmd, args, { timeout: 0 });
+    cp.stdout.setEncoding('utf8');
+    cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); });
+    cp.stderr.setEncoding('utf8');
+    cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); });
+
+    cp.on('error', function (error) {
+        debug('%s : child process errored %s', tag, error.message);
+        callback(error);
+    });
+
+    cp.on('exit', function (code, signal) {
+        debug('%s : child process exited. code: %d signal: %d', tag, code, signal);
+        if (signal) return callback(new Error('Exited with signal ' + signal));
+        if (code !== 0) return callback(new Error('Exited with code ' + code));
+
+        callback(null);
+    });
+}
+
+function ensureVersion(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!args.data || !args.data.boxVersionsUrl) return callback(new Error('No boxVersionsUrl specified'));
+
+    if (args.sourceTarballUrl) return callback(null, args);
+
+    superagent.get(args.data.boxVersionsUrl).end(function (error, result) {
+        if (error && !error.response) return callback(error);
+        if (result.statusCode !== 200) return callback(new Error(util.format('Bad status: %s %s', result.statusCode, result.text)));
+
+        var versions = safe.JSON.parse(result.text);
+
+        if (!versions || typeof versions !== 'object') return callback(new Error('versions is not in valid format:' + safe.error));
+
+        var latestVersion = Object.keys(versions).sort(semver.compare).pop();
+        debug('ensureVersion: Latest version is %s etag:%s', latestVersion, result.header['etag']);
+
+        if (!versions[latestVersion]) return callback(new Error('No version available'));
+        if (!versions[latestVersion].sourceTarballUrl) return callback(new Error('No sourceTarballUrl specified'));
+
+        args.sourceTarballUrl = versions[latestVersion].sourceTarballUrl;
+        args.data.version = latestVersion;
+
+        callback(null, args);
+    });
+}
+
+function provision(args, callback) {
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (process.env.NODE_ENV === 'test') return callback(null);
+
+    ensureVersion(args, function (error, result) {
+        if (error) return callback(error);
+
+        var pargs = [ INSTALLER_CMD ];
+        pargs.push('--sourcetarballurl', result.sourceTarballUrl);
+        pargs.push('--data', JSON.stringify(result.data));
+
+        debug('provision: calling with args %j', pargs);
+
+        // sudo is required for update()
+        spawn('provision', SUDO, pargs, callback);
+    });
+}
+

installer/src/scripts/installer.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly BOX_SRC_DIR=/home/yellowtent/box
+readonly DATA_DIR=/home/yellowtent/data
+
+readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly json="${script_dir}/../../node_modules/.bin/json"
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
+
+readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
+
+# create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@
+(echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh
+chmod +x /home/yellowtent/provision.sh
+
+arg_source_tarball_url=""
+arg_data=""
+
+args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --sourcetarballurl) arg_source_tarball_url="$2";;
+    --data) arg_data="$2";;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+
+    shift 2
+done
+
+box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
+echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}"
+
+while true; do
+    if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi
+    echo "Failed to download source tarball, trying again"
+    sleep 5
+done
+while true; do
+    # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
+    if cd "${box_src_tmp_dir}" && npm rebuild; then break; fi
+    echo "Failed to rebuild, trying again"
+    sleep 5
+done
+
+if [[ "${is_update}" == "yes" ]]; then
+    echo "Setting up update splash screen"
+    "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code
+    ${BOX_SRC_DIR}/setup/stop.sh # stop the old code
+fi
+
+# switch the codes
+rm -rf "${BOX_SRC_DIR}"
+mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}"
+chown -R yellowtent.yellowtent "${BOX_SRC_DIR}"
+
+# create a start file for testing. %q escapes args
+(echo -e "#!/bin/bash\n"; printf "%q " "${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}") > /home/yellowtent/setup_start.sh
+chmod +x /home/yellowtent/setup_start.sh
+
+echo "Calling box setup script"
+"${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}"
+

installer/src/server.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+
+/* jslint node: true */
+
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    debug = require('debug')('installer:server'),
+    express = require('express'),
+    fs = require('fs'),
+    http = require('http'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    installer = require('./installer.js'),
+    json = require('body-parser').json,
+    lastMile = require('connect-lastmile'),
+    morgan = require('morgan'),
+    superagent = require('superagent');
+
+exports = module.exports = {
+    start: start,
+    stop: stop
+};
+
+var PROVISION_CONFIG_FILE = '/root/provision.json';
+var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
+
+var gHttpServer = null; // update server; used for updates
+
+function provisionDigitalOcean(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
+        if (error || result.statusCode !== 200) {
+            console.error('Error getting metadata', error);
+            return callback(new Error('Error getting metadata'));
+        }
+
+        var userData = JSON.parse(result.body.user_data);
+
+        installer.provision(userData, callback);
+    });
+}
+
+function provisionLocal(callback) {
+    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
+
+    if (!fs.existsSync(PROVISION_CONFIG_FILE)) {
+        console.error('No provisioning data found at %s', PROVISION_CONFIG_FILE);
+        return callback(new Error('No provisioning data found'));
+    }
+
+    var userData = require(PROVISION_CONFIG_FILE);
+
+    installer.provision(userData, callback);
+}
+
+function update(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (!req.body.sourceTarballUrl || typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided'));
+    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
+
+    debug('provision: received from box %j', req.body);
+
+    installer.provision(req.body, function (error) {
+        if (error) console.error(error);
+    });
+
+    next(new HttpSuccess(202, { }));
+}
+
+function startUpdateServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting update server');
+
+    var app = express();
+
+    var router = new express.Router();
+
+    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
+
+    app.use(json({ strict: true }))
+       .use(router)
+       .use(lastMile());
+
+    router.post('/api/v1/installer/update', update);
+
+    gHttpServer = http.createServer(app);
+    gHttpServer.on('error', console.error);
+
+    gHttpServer.listen(2020, '127.0.0.1', callback);
+}
+
+function stopUpdateServer(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Stopping update server');
+
+    if (!gHttpServer) return callback(null);
+
+    gHttpServer.close(callback);
+    gHttpServer = null;
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var actions;
+
+    if (process.env.PROVISION === 'local') {
+        debug('Starting Installer in selfhost mode');
+
+        actions = [
+            startUpdateServer,
+            provisionLocal
+        ];
+    } else {    // current fallback, should be 'digitalocean' eventually, see initializeBaseUbuntuImage.sh
+        debug('Starting Installer in managed mode');
+
+        actions = [
+            startUpdateServer,
+            provisionDigitalOcean
+        ];
+    }
+
+    async.series(actions, callback);
+}
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        stopUpdateServer
+    ], callback);
+}
+
+if (require.main === module) {
+    start(function (error) {
+        if (error) console.error(error);
+    });
+}

installer/src/test/installer-test.js

@@ -0,0 +1,179 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var expect = require('expect.js'),
+    fs = require('fs'),
+    path = require('path'),
+    nock = require('nock'),
+    os = require('os'),
+    request = require('superagent'),
+    server = require('../server.js'),
+    installer = require('../installer.js'),
+    _ = require('lodash');
+
+var EXTERNAL_SERVER_URL = 'https://localhost:4443';
+var INTERNAL_SERVER_URL = 'http://localhost:2020';
+var APPSERVER_ORIGIN = 'http://appserver';
+var FQDN = os.hostname();
+
+describe('Server', function () {
+    this.timeout(5000);
+
+    before(function (done) {
+        var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string
+        var scope = nock('http://169.254.169.254')
+            .persist()
+            .get('/metadata/v1.json')
+            .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' });
+        done();
+    });
+
+    after(function (done) {
+        nock.cleanAll();
+        done();
+    });
+
+    describe('starts and stop', function () {
+        it('starts', function (done) {
+            server.start(done);
+        });
+
+        it('stops', function (done) {
+            server.stop(done);
+        });
+    });
+
+    describe('update (internal server)', function () {
+        before(function (done) {
+            server.start(done);
+        });
+        after(function (done) {
+            server.stop(done);
+        });
+
+        it('does not respond to provision', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        it('does not respond to restore', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(404);
+                done();
+            });
+        });
+
+        var data = {
+            sourceTarballUrl: "https://foo.tar.gz",
+
+            data: {
+                token: 'sometoken',
+                apiServerOrigin: APPSERVER_ORIGIN,
+                webServerOrigin: 'https://somethingelse.com',
+                fqdn: 'www.something.com',
+                tlsKey: 'key',
+                tlsCert: 'cert',
+                boxVersionsUrl: 'https://versions.json',
+                version: '0.1'
+            }
+        };
+
+        Object.keys(data).forEach(function (key) {
+            it('fails due to missing ' + key, function (done) {
+                var dataCopy = _.merge({ }, data);
+                delete dataCopy[key];
+
+                request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) {
+                    expect(error).to.not.be.ok();
+                    expect(result.statusCode).to.equal(400);
+                    done();
+                });
+            });
+        });
+
+        it('succeeds', function (done) {
+            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) {
+                expect(error).to.not.be.ok();
+                expect(result.statusCode).to.equal(202);
+                done();
+            });
+        });
+    });
+
+    describe('ensureVersion', function () {
+        before(function () {
+            process.env.NODE_ENV = undefined;
+        });
+
+        after(function () {
+            process.env.NODE_ENV = 'test';
+        });
+
+        it ('fails without data', function (done) {
+            installer._ensureVersion({}, function (error) {
+                expect(error).to.be.an(Error);
+                done();
+            });
+        });
+
+        it ('fails without boxVersionsUrl', function (done) {
+            installer._ensureVersion({ data: {}}, function (error) {
+                expect(error).to.be.an(Error);
+                done();
+            });
+        });
+
+        it ('succeeds with sourceTarballUrl', function (done) {
+            var data = {
+                sourceTarballUrl: 'sometarballurl',
+                data: {
+                    boxVersionsUrl: 'http://foobar/versions.json'
+                }
+            };
+
+            installer._ensureVersion(data, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result).to.eql(data);
+                done();
+            });
+        });
+
+        it ('succeeds without sourceTarballUrl', function (done) {
+            var versions = {
+                '0.1.0': {
+                    sourceTarballUrl: 'sometarballurl1'
+                },
+                '0.2.0': {
+                    sourceTarballUrl: 'sometarballurl2'
+                }
+            };
+
+            var scope = nock('http://foobar')
+                .get('/versions.json')
+                .reply(200, JSON.stringify(versions), { 'Content-Type': 'application/json' });
+
+            var data = {
+                data: {
+                    boxVersionsUrl: 'http://foobar/versions.json'
+                }
+            };
+
+            installer._ensureVersion(data, function (error, result) {
+                expect(error).to.equal(null);
+                expect(result.sourceTarballUrl).to.equal(versions['0.2.0'].sourceTarballUrl);
+                expect(result.data.boxVersionsUrl).to.equal(data.data.boxVersionsUrl);
+                done();
+            });
+        });
+    });
+});
+

installer/systemd/box-setup.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly USER_HOME="/home/yellowtent"
+readonly APPS_SWAP_FILE="/apps.swap"
+readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups
+readonly USER_DATA_FILE="/root/user_data.img"
+readonly USER_DATA_DIR="/home/yellowtent/data"
+
+# detect device
+if [[ -b "/dev/vda1" ]]; then
+    disk_device="/dev/vda1"
+fi
+
+if [[ -b "/dev/xvda1" ]]; then
+    disk_device="/dev/xvda1"
+fi
+
+# all sizes are in mb
+readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
+readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
+readonly app_count=$((${physical_memory} / 200)) # estimated app count
+readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
+readonly disk_size=$((disk_size_gb * 1024))
+readonly backup_swap_size=1024
+readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
+readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
+
+echo "Disk device: ${disk_device}"
+echo "Physical memory: ${physical_memory}"
+echo "Estimated app count: ${app_count}"
+echo "Disk size: ${disk_size}"
+
+# Allocate two sets of swap files - one for general app usage and another for backup
+# The backup swap is setup for swap on the fly by the backup scripts
+if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
+    echo "Creating Apps swap file of size ${swap_size}M"
+    fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
+    chmod 600 "${APPS_SWAP_FILE}"
+    mkswap "${APPS_SWAP_FILE}"
+    swapon "${APPS_SWAP_FILE}"
+    echo "${APPS_SWAP_FILE}  none  swap  sw  0 0" >> /etc/fstab
+else
+    echo "Apps Swap file already exists"
+fi
+
+if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then
+    echo "Creating Backup swap file of size ${backup_swap_size}M"
+    fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}"
+    chmod 600 "${BACKUP_SWAP_FILE}"
+    mkswap "${BACKUP_SWAP_FILE}"
+else
+    echo "Backups Swap file already exists"
+fi
+
+echo "Resizing data volume"
+home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved))
+echo "Resizing up btrfs user data to size ${home_data_size}M"
+umount "${USER_DATA_DIR}" || true
+# Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data
+# fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
+truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
+mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
+btrfs filesystem resize max "${USER_DATA_DIR}"
+

janitor.js

@@ -1,70 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var assert = require('assert'),
-    debug = require('debug')('box:janitor'),
-    async = require('async'),
-    tokendb = require('./src/tokendb.js'),
-    authcodedb = require('./src/authcodedb.js'),
-    database = require('./src/database.js');
-
-var TOKEN_CLEANUP_INTERVAL = 30000;
-
-function initialize(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.initialize
-    ], callback);
-}
-
-function cleanupExpiredTokens(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired tokens.', result);
-
-        callback(null);
-    });
-}
-
-function cleanupExpiredAuthCodes(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    authcodedb.delExpired(function (error, result) {
-        if (error) return callback(error);
-
-        debug('Cleaned up %s expired authcodes.', result);
-
-        callback(null);
-    });
-}
-
-function run() {
-    cleanupExpiredTokens(function (error) {
-        if (error) console.error(error);
-
-        cleanupExpiredAuthCodes(function (error) {
-            if (error) console.error(error);
-
-            setTimeout(run, TOKEN_CLEANUP_INTERVAL);
-        });
-    });
-}
-
-if (require.main === module) {
-    initialize(function (error) {
-        if (error) {
-            console.error('janitor task exiting with error', error);
-            process.exit(1);
-        }
-
-        run();
-    });
-}
-

migrations/20151013073519-apps-add-oauthProxy.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151015232915-clients-add-type.js

@@ -0,0 +1,17 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'DELETE FROM clients'),
+        db.runSql.bind(db, 'ALTER TABLE clients ADD COLUMN type VARCHAR(16) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE clients DROP COLUMN type', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151016131005-apps-rename-accessRestriction.js

@@ -0,0 +1,17 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestriction accessRestrictionJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE accessRestrictionJson accessRestriction VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20151113091257-apps-alter-manifestJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(2048)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20151126111337-apps-alter-jsons.js

@@ -0,0 +1,19 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson TEXT'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson TEXT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY accessRestrictionJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY lastBackupConfigJson VARCHAR(2048)'),
+        db.runSql.bind(db, 'ALTER TABLE apps MODIFY oldConfigJson VARCHAR(2048)')
+    ], callback);
+};

migrations/20160119113143-users-add-displayName.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN displayName VARCHAR(512) DEFAULT ""', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN displayName', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160205135326-apps-add-memoryLimit.js

@@ -0,0 +1,15 @@
+dbm = dbm || require('db-migrate');
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN memoryLimit BIGINT DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN memoryLimit', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208031241-groups-add-table.js

@@ -0,0 +1,21 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE groups(" +
+                "id VARCHAR(128) NOT NULL UNIQUE," +
+                "name VARCHAR(128) NOT NULL UNIQUE," +
+                "PRIMARY KEY(id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208100000-groupMembers-add-table.js

@@ -0,0 +1,22 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = "CREATE TABLE IF NOT EXISTS groupMembers(" +
+    			"groupId VARCHAR(128) NOT NULL," +
+    			"userId VARCHAR(128) NOT NULL," +
+    			"FOREIGN KEY(groupId) REFERENCES groups(id)," +
+    			"FOREIGN KEY(userId) REFERENCES users(id));";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE groupMembers', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160208164735-groups-add-admin.js

@@ -0,0 +1,30 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var async = require('async');
+
+var ADMIN_GROUP_ID = 'admin'; // see groups.js
+
+exports.up = function(db, callback) {
+	async.series([
+		db.runSql.bind(db, 'START TRANSACTION;'),
+		db.runSql.bind(db, 'INSERT INTO groups (id, name) VALUES (?, ?)', [ ADMIN_GROUP_ID, 'admin' ]),
+		function migrateAdminFlag(done) {
+			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
+				if (error) return done(error);
+
+				console.dir(results);
+
+				async.eachSeries(results, function (r, next) {
+					db.runSql('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ ADMIN_GROUP_ID, r.id ], next);
+				}, done);
+			});
+		},
+		db.runSql.bind(db, 'ALTER TABLE users DROP COLUMN admin'),
+		db.runSql.bind(db, 'COMMIT')
+	], callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20160307172229-backups-add-table.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE backups(" +
+            "filename VARCHAR(128) NOT NULL," +
+            "creationTime TIMESTAMP," +
+            "version VARCHAR(128) NOT NULL," +
+            "type VARCHAR(16) NOT NULL," +
+            "dependsOn VARCHAR(4096)," +
+            "state VARCHAR(16) NOT NULL," +
+            "PRIMARY KEY (filename))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE backups', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -18,8 +18,20 @@ CREATE TABLE IF NOT EXISTS users(
     createdAt VARCHAR(512) NOT NULL,
     modifiedAt VARCHAR(512) NOT NULL,
     admin INTEGER NOT NULL,
+    displayName VARCHAR(512) DEFAULT '',
     PRIMARY KEY(id));
 
+CREATE TABLE IF NOT EXISTS groups(
+    id VARCHAR(128) NOT NULL UNIQUE,
+    username VARCHAR(254) NOT NULL UNIQUE,
+    PRIMARY KEY(id));
+
+CREATE TABLE IF NOT EXISTS groupMembers(
+    groupId VARCHAR(128) NOT NULL,
+    userId VARCHAR(128) NOT NULL,
+    FOREIGN KEY(groupId) REFERENCES groups(id),
+    FOREIGN KEY(userId) REFERENCES users(id));
+
 CREATE TABLE IF NOT EXISTS tokens(
     accessToken VARCHAR(128) NOT NULL UNIQUE,
     identifier VARCHAR(128) NOT NULL,
@@ -29,8 +41,9 @@ CREATE TABLE IF NOT EXISTS tokens(
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
-    id VARCHAR(128) NOT NULL UNIQUE,
+    id VARCHAR(128) NOT NULL UNIQUE, // prefixed with cid- to identify token easily in auth routes
     appId VARCHAR(128) NOT NULL,
+    type VARCHAR(16) NOT NULL,
     clientSecret VARCHAR(512) NOT NULL,
     redirectURI VARCHAR(512) NOT NULL,
     scope VARCHAR(512) NOT NULL,
@@ -44,15 +57,20 @@ CREATE TABLE IF NOT EXISTS apps(
     runState VARCHAR(512),
     health VARCHAR(128),
     containerId VARCHAR(128),
-    manifestJson VARCHAR(2048),
+    manifestJson TEXT,
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestriction VARCHAR(512),
+    accessRestrictionJson TEXT,
+    oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    memoryLimit BIGINT DEFAULT 0,
 
     lastBackupId VARCHAR(128),
-    lastBackupConfigJson VARCHAR(2048), // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+
+    oldConfigJson TEXT, // used to pass old config for apptask
+
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS appPortBindings(
@@ -80,3 +98,12 @@ CREATE TABLE IF NOT EXISTS appAddonConfigs(
     value VARCHAR(512) NOT NULL,
     FOREIGN KEY(appId) REFERENCES apps(id));
 
+CREATE TABLE IF NOT EXISTS backups(
+    filename VARCHAR(128) NOT NULL, /* s3 url */
+    creationTime TIMESTAMP,
+    version VARCHAR(128) NOT NULL, /* app version or box version */
+    type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
+    dependsOn VARCHAR(4096), /* comma separate list of objects this backup depends on */
+    state VARCHAR(16) NOT NULL,
+
+    PRIMARY KEY (filename));

oauthproxy.js

@@ -1,185 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-require('supererror')({ splatchError: true });
-
-var express = require('express'),
-    url = require('url'),
-    uuid = require('node-uuid'),
-    async = require('async'),
-    superagent = require('superagent'),
-    assert = require('assert'),
-    debug = require('debug')('box:proxy'),
-    proxy = require('proxy-middleware'),
-    session = require('cookie-session'),
-    database = require('./src/database.js'),
-    appdb = require('./src/appdb.js'),
-    clientdb = require('./src/clientdb.js'),
-    config = require('./src/config.js'),
-    http = require('http');
-
-// Allow self signed certs!
-process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
-
-var gSessions = {};
-var gProxyMiddlewareCache = {};
-var gApp = express();
-var gHttpServer = http.createServer(gApp);
-
-var CALLBACK_URI = '/callback';
-var PORT = 4000;
-
-function startServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    gHttpServer.on('error', console.error);
-
-    gApp.use(session({
-        keys: ['blue', 'cheese', 'is', 'something']
-    }));
-
-    // ensure we have a in memory store for the session to cache client information
-    gApp.use(function (req, res, next) {
-        assert.strictEqual(typeof req.session, 'object');
-
-        if (!req.session.id || !gSessions[req.session.id]) {
-            req.session.id = uuid.v4();
-            gSessions[req.session.id] = {};
-        }
-
-        // attach the session data to the requeset
-        req.sessionData = gSessions[req.session.id];
-
-        next();
-    });
-
-    gApp.use(function verifySession(req, res, next) {
-        assert.strictEqual(typeof req.sessionData, 'object');
-
-        if (!req.sessionData.accessToken) {
-            req.authenticated = false;
-            return next();
-        }
-
-        superagent.get(config.adminOrigin() + '/api/v1/profile').query({ access_token: req.sessionData.accessToken}).end(function (error, result) {
-            if (error) {
-                console.error(error);
-                req.authenticated = false;
-            } else if (result.statusCode !== 200) {
-                req.sessionData.accessToken = null;
-                req.authenticated = false;
-            } else {
-                req.authenticated = true;
-            }
-
-            next();
-        });
-    });
-
-    gApp.use(function (req, res, next) {
-        // proceed if we are authenticated
-        if (req.authenticated) return next();
-
-        if (req.path === CALLBACK_URI && req.sessionData.returnTo) {
-            // exchange auth code for an access token
-            var query = {
-                response_type: 'token',
-                client_id: req.sessionData.clientId
-            };
-
-            var data = {
-                grant_type: 'authorization_code',
-                code: req.query.code,
-                redirect_uri: req.sessionData.returnTo,
-                client_id: req.sessionData.clientId,
-                client_secret: req.sessionData.clientSecret
-            };
-
-            superagent.post(config.adminOrigin() + '/api/v1/oauth/token').query(query).send(data).end(function (error, result) {
-                if (error) {
-                    console.error(error);
-                    return res.send(500, 'Unable to contact the oauth server.');
-                }
-                if (result.statusCode !== 200) {
-                    console.error('Failed to exchange auth code for a token.', result.statusCode, result.body);
-                    return res.send(500, 'Failed to exchange auth code for a token.');
-                }
-
-                req.sessionData.accessToken = result.body.access_token;
-
-                debug('user verified.');
-
-                // now redirect to the actual initially requested URL
-                res.redirect(req.sessionData.returnTo);
-            });
-        } else {
-            var port = parseInt(req.headers['x-cloudron-proxy-port'], 10);
-
-            if (!Number.isFinite(port)) {
-                console.error('Failed to parse nginx proxy header to get app port.');
-                return res.send(500, 'Routing error. No forwarded port.');
-            }
-
-            debug('begin verifying user for app on port %s.', port);
-
-            appdb.getByHttpPort(port, function (error, result) {
-                if (error) {
-                    console.error('Unknown app.', error);
-                    return res.send(500, 'Unknown app.');
-                }
-
-                clientdb.getByAppId('proxy-' + result.id, function (error, result) {
-                    if (error) {
-                        console.error('Unkonwn OAuth client.', error);
-                        return res.send(500, 'Unknown OAuth client.');
-                    }
-
-                    req.sessionData.port = port;
-                    req.sessionData.returnTo = result.redirectURI + req.path;
-                    req.sessionData.clientId = result.id;
-                    req.sessionData.clientSecret = result.clientSecret;
-
-                    var callbackUrl = result.redirectURI + CALLBACK_URI;
-                    var scope = 'profile,roleUser';
-                    var oauthLogin = config.adminOrigin() + '/api/v1/oauth/dialog/authorize?response_type=code&client_id=' + result.id + '&redirect_uri=' + callbackUrl + '&scope=' + scope;
-
-                    debug('begin OAuth flow for client %s.', result.name);
-
-                    // begin the OAuth flow
-                    res.redirect(oauthLogin);
-                });
-            });
-        }
-    });
-
-    gApp.use(function (req, res, next) {
-        var port = req.sessionData.port;
-
-        debug('proxy request for port %s with path %s.', port, req.path);
-
-        var proxyMiddleware = gProxyMiddlewareCache[port];
-        if (!proxyMiddleware) {
-            console.log('Adding proxy middleware for port %d', port);
-
-            proxyMiddleware = proxy(url.parse('http://127.0.0.1:' + port));
-            gProxyMiddlewareCache[port] = proxyMiddleware;
-        }
-
-        proxyMiddleware(req, res, next);
-    });
-
-    gHttpServer.listen(PORT, callback);
-}
-
-async.series([
-    database.initialize,
-    startServer
-], function (error) {
-    if (error) {
-        console.error('Failed to start proxy server.', error);
-        process.exit(1);
-    }
-
-    console.log('Proxy server listening...');
-});

package.json

@@ -10,15 +10,15 @@
     "type": "git"
   },
   "engines": [
-    "node >= 0.12.0"
+    "node >=4.0.0 <=4.1.1"
   ],
-  "bin": {
-    "cloudron": "./app.js"
-  },
   "dependencies": {
     "async": "^1.2.1",
+    "attempt": "^1.0.1",
+    "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "cloudron-manifestformat": "^1.6.0",
+    "bytes": "^2.1.0",
+    "cloudron-manifestformat": "^2.3.0",
     "connect-ensure-login": "^0.1.1",
     "connect-lastmile": "0.0.13",
     "connect-timeout": "^1.5.0",
@@ -42,51 +42,57 @@
     "multiparty": "^4.1.2",
     "mysql": "^2.7.0",
     "native-dns": "^0.7.0",
+    "node-df": "^0.1.1",
     "node-uuid": "^1.4.3",
     "nodemailer": "^1.3.0",
     "nodemailer-smtp-transport": "^1.0.3",
     "oauth2orize": "^1.0.1",
     "once": "^1.3.2",
+    "parse-links": "^0.1.0",
     "passport": "^0.2.2",
     "passport-http": "^0.2.2",
     "passport-http-bearer": "^1.0.1",
     "passport-local": "^1.0.0",
     "passport-oauth2-client-password": "^0.1.2",
-    "password-generator": "^1.0.0",
+    "password-generator": "^2.0.2",
     "proxy-middleware": "^0.13.0",
-    "safetydance": "0.0.19",
+    "safetydance": "^0.1.1",
     "semver": "^4.3.6",
     "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
-    "superagent": "~0.21.0",
+    "superagent": "^1.5.0",
-    "supererror": "^0.7.0",
+    "supererror": "^0.7.1",
-    "supervisord-eventlistener": "^0.1.0",
     "tail-stream": "https://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz",
+    "tldjs": "^1.6.2",
     "underscore": "^1.7.0",
+    "ursa": "^0.9.1",
     "valid-url": "^1.0.9",
-    "validator": "^3.30.0"
+    "validator": "^4.4.0",
+    "x509": "^0.2.2"
   },
   "devDependencies": {
     "apidoc": "*",
-    "aws-sdk": "^2.1.10",
     "bootstrap-sass": "^3.3.3",
+    "deep-extend": "^0.4.1",
     "del": "^1.1.1",
     "expect.js": "*",
     "gulp": "^3.8.11",
     "gulp-autoprefixer": "^2.3.0",
     "gulp-concat": "^2.4.3",
+    "gulp-cssnano": "^2.1.0",
     "gulp-ejs": "^1.0.0",
-    "gulp-minify-css": "^1.1.3",
     "gulp-sass": "^2.0.1",
     "gulp-serve": "^1.0.0",
     "gulp-sourcemaps": "^1.5.2",
     "gulp-uglify": "^1.1.0",
     "hock": "~1.2.0",
     "istanbul": "*",
+    "js2xmlparser": "^1.0.0",
     "mocha": "*",
-    "nock": "^2.6.0",
+    "nock": "^3.4.0",
     "node-sass": "^3.0.0-alpha.0",
-    "redis": "^0.12.1",
+    "redis": "^2.4.2",
+    "request": "^2.65.0",
     "sinon": "^1.12.2",
     "yargs": "^3.15.0"
   },

scripts/createReleaseTarball

@@ -0,0 +1,123 @@
+#!/bin/bash
+
+set -eu
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
+# brew install gnu-getopt to get the GNU getopt on OS X
+[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
+readonly GNU_GETOPT
+
+args=$(${GNU_GETOPT} -o "" -l "revision:,output:,publish,no-upload" -n "$0" -- "$@")
+eval set -- "${args}"
+
+readonly RELEASE_TOOL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../release" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
+delete_bundle="yes"
+commitish="HEAD"
+publish="no"
+upload="yes"
+bundle_file=""
+
+while true; do
+    case "$1" in
+    --revision) commitish="$2"; shift 2;;
+    --output) bundle_file="$2"; delete_bundle="no"; shift 2;;
+    --no-upload) upload="no"; shift;;
+    --publish) publish="yes"; shift;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+if [[ "${upload}" == "no" && "${publish}" == "yes" ]]; then
+    echo "Cannot publish without uploading"
+    exit 1
+fi
+
+readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint?
+
+assertNotEmpty AWS_DEV_ACCESS_KEY
+assertNotEmpty AWS_DEV_SECRET_KEY
+
+if ! $(cd "${SOURCE_DIR}" && git diff --exit-code >/dev/null); then
+    echo "You have local changes, stash or commit them to proceed"
+    exit 1
+fi
+
+if [[ "$(node --version)" != "v4.1.1" ]]; then
+    echo "This script requires node 4.1.1"
+    exit 1
+fi
+
+version=$(cd "${SOURCE_DIR}" && git rev-parse "${commitish}")
+bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
+[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${version}.tar.gz"
+
+chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
+echo "Checking out code [${version}] into ${bundle_dir}"
+(cd "${SOURCE_DIR}" && git archive --format=tar ${version} | (cd "${bundle_dir}" && tar xf -))
+
+if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then
+    echo "Reusing dev modules from cache"
+    cp -r "${TMPDIR}/boxtarball.cache/node_modules-all/." "${bundle_dir}/node_modules"
+else
+    echo "Installing modules with dev dependencies"
+    (cd "${bundle_dir}" && npm install)
+
+    echo "Caching dev dependencies"
+    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-all"
+    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-all/"
+    cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all"
+fi
+
+echo "Building webadmin assets"
+(cd "${bundle_dir}" && gulp)
+
+echo "Remove intermediate files required at build-time only"
+rm -rf "${bundle_dir}/node_modules/"
+rm -rf "${bundle_dir}/webadmin/src"
+rm -rf "${bundle_dir}/gulpfile.js"
+
+if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then
+    echo "Reusing prod modules from cache"
+    cp -r "${TMPDIR}/boxtarball.cache/node_modules-prod/." "${bundle_dir}/node_modules"
+else
+    echo "Installing modules for production"
+    (cd "${bundle_dir}" && npm install --production --no-optional)
+
+    echo "Caching prod dependencies"
+    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-prod"
+    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-prod/"
+    cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod"
+fi
+
+echo "Create final tarball"
+(cd "${bundle_dir}" && tar czf "${bundle_file}" .)
+echo "Cleaning up ${bundle_dir}"
+rm -rf "${bundle_dir}"
+
+if [[ "${upload}" == "yes" ]]; then
+    echo "Uploading bundle to S3"
+    # That special header is needed to allow access with singed urls created with different aws credentials than the ones the file got uploaded
+    s3cmd --multipart-chunk-size-mb=5 --ssl --acl-public --access_key="${AWS_DEV_ACCESS_KEY}" --secret_key="${AWS_DEV_SECRET_KEY}" --no-mime-magic put "${bundle_file}" "s3://dev-cloudron-releases/box-${version}.tar.gz"
+
+    versions_file_url="https://dev-cloudron-releases.s3.amazonaws.com/box-${version}.tar.gz"
+    echo "The URL for the versions file is: ${versions_file_url}"
+
+    if [[ "${publish}" == "yes" ]]; then
+        echo "Publishing to dev"
+        ${RELEASE_TOOL_DIR}/release create --env dev --code "${versions_file_url}"
+    fi
+fi
+
+if [[ "${delete_bundle}" == "no" ]]; then
+    echo "Tarball preserved at ${bundle_file}"
+else
+    rm "${bundle_file}"
+fi
+

setup/DESIGN.md

@@ -16,7 +16,7 @@ and replace it with a new one for an update.
 
 Because we do not package things as Docker yet, we should be careful
 about the code here. We have to expect remains of an older setup code.
-For example, older supervisor or nginx configs might be around.
+For example, older systemd or nginx configs might be around.
 
 The config directory is _part_ of the container and is not a VOLUME.
 Which is to say that the files will be nuked from one update to the next.
@@ -40,7 +40,7 @@ version (see below) or the mysql/postgresql data etc.
 
   * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
 
-  * supervisor is then started
+  * box services are then started
 
 setup_infra.sh
 This setups containers like graphite, mail and the addons containers.

setup/INFRA_VERSION

@@ -3,15 +3,21 @@
 # If you change the infra version, be sure to put a warning
 # in the change log
 
-INFRA_VERSION=8
+INFRA_VERSION=23
 
 # WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
 # These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.3.1
+BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.3.2
+MYSQL_IMAGE=cloudron/mysql:0.10.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.3.1
+POSTGRESQL_IMAGE=cloudron/postgresql:0.8.0
-MONGODB_IMAGE=cloudron/mongodb:0.3.1
+MONGODB_IMAGE=cloudron/mongodb:0.8.0
-REDIS_IMAGE=cloudron/redis:0.3.1 # if you change this, fix src/addons.js as well
+REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.3.1
+MAIL_IMAGE=cloudron/mail:0.9.0
-GRAPHITE_IMAGE=cloudron/graphite:0.3.3
+GRAPHITE_IMAGE=cloudron/graphite:0.8.0
 
+MYSQL_REPO=cloudron/mysql
+POSTGRESQL_REPO=cloudron/postgresql
+MONGODB_REPO=cloudron/mongodb
+REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
+MAIL_REPO=cloudron/mail
+GRAPHITE_REPO=cloudron/graphite

setup/argparser.sh

@@ -11,11 +11,16 @@ arg_is_custom_domain="false"
 arg_restore_key=""
 arg_restore_url=""
 arg_retire="false"
+arg_tls_config=""
 arg_tls_cert=""
 arg_tls_key=""
 arg_token=""
 arg_version=""
 arg_web_server_origin=""
+arg_backup_config=""
+arg_dns_config=""
+arg_update_config=""
+arg_provider=""
 
 args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
 eval set -- "${args}"
@@ -28,19 +33,33 @@ while true; do
         ;;
     --data)
         # only read mandatory non-empty parameters here
-        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_token arg_is_custom_domain arg_box_versions_url arg_version <<EOF
+        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_is_custom_domain arg_box_versions_url arg_version <<EOF
-        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn token isCustomDomain boxVersionsUrl version | tr '\n' ' ')
+        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn isCustomDomain boxVersionsUrl version | tr '\n' ' ')
 EOF
         # read possibly empty parameters here
         arg_tls_cert=$(echo "$2" | $json tlsCert)
         arg_tls_key=$(echo "$2" | $json tlsKey)
+        arg_token=$(echo "$2" | $json token)
+        arg_provider=$(echo "$2" | $json provider)
 
-        arg_restore_url=$(echo "$2" | $json restoreUrl)
+        arg_tls_config=$(echo "$2" | $json tlsConfig)
+        [[ "${arg_tls_config}" == "null" ]] && arg_tls_config=""
+
+        arg_restore_url=$(echo "$2" | $json restore.url)
         [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
 
-        arg_restore_key=$(echo "$2" | $json restoreKey)
+        arg_restore_key=$(echo "$2" | $json restore.key)
         [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
 
+        arg_backup_config=$(echo "$2" | $json backupConfig)
+        [[ "${arg_backup_config}" == "null" ]] && arg_backup_config=""
+
+        arg_dns_config=$(echo "$2" | $json dnsConfig)
+        [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
+
+        arg_update_config=$(echo "$2" | $json updateConfig)
+        [[ "${arg_update_config}" == "null" ]] && arg_update_config=""
+
         shift 2
         ;;
     --) break;;
@@ -58,5 +77,7 @@ echo "restore url: ${arg_restore_url}"
 echo "tls cert: ${arg_tls_cert}"
 echo "tls key: ${arg_tls_key}"
 echo "token: ${arg_token}"
+echo "tlsConfig: ${arg_tls_config}"
 echo "version: ${arg_version}"
 echo "web server: ${arg_web_server_origin}"
+echo "provider: ${arg_provider}"

setup/container.sh

@@ -13,22 +13,24 @@ readonly DATA_DIR="/home/yellowtent/data"
 rm -rf "${CONFIG_DIR}"
 sudo -u yellowtent mkdir "${CONFIG_DIR}"
 
-########## logrotate (default ubuntu runs this daily)
+########## systemd
-rm -rf /etc/logrotate.d/*
+rm -f /etc/systemd/system/janitor.*
-cp -r "${container_files}/logrotate/." /etc/logrotate.d/
+cp -r "${container_files}/systemd/." /etc/systemd/system/
-
+systemctl daemon-reload
-########## supervisor
+systemctl enable cloudron.target
-rm -rf /etc/supervisor/*
-cp -r "${container_files}/supervisor/." /etc/supervisor/
 
 ########## sudoers
-rm /etc/sudoers.d/*
+rm -f /etc/sudoers.d/yellowtent
 cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
 
 ########## collectd
 rm -rf /etc/collectd
 ln -sfF "${DATA_DIR}/collectd" /etc/collectd
 
+########## apparmor docker profile
+cp "${container_files}/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
+systemctl restart apparmor
+
 ########## nginx
 # link nginx config to system config
 unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx

setup/container/docker-cloudron-app.apparmor

@@ -0,0 +1,32 @@
+#include <tunables/global>
+
+
+profile docker-cloudron-app flags=(attach_disconnected,mediate_deleted) {
+
+  #include <abstractions/base>
+
+  ptrace peer=@{profile_name},
+
+  network,
+  capability,
+  file,
+  umount,
+
+  deny @{PROC}/sys/fs/** wklx,
+  deny @{PROC}/sysrq-trigger rwklx,
+  deny @{PROC}/mem rwklx,
+  deny @{PROC}/kmem rwklx,
+  deny @{PROC}/sys/kernel/[^s][^h][^m]* wklx,
+  deny @{PROC}/sys/kernel/*/** wklx,
+
+  deny mount,
+
+  deny /sys/[^f]*/** wklx,
+  deny /sys/f[^s]*/** wklx,
+  deny /sys/fs/[^c]*/** wklx,
+  deny /sys/fs/c[^g]*/** wklx,
+  deny /sys/fs/cg[^r]*/** wklx,
+  deny /sys/firmware/efi/efivars/** rwklx,
+  deny /sys/kernel/security/** rwklx,
+}
+

setup/container/logrotate/cloudron

@@ -1,6 +0,0 @@
-/var/log/cloudron/*log {
-    missingok
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/logrotate/supervisor

@@ -1,7 +0,0 @@
-/var/log/supervisor/*log {
-    missingok
-    copytruncate
-    notifempty
-    size 100k
-    nocompress
-}

setup/container/sudoers

@@ -1,3 +1,6 @@
+# sudo logging breaks journalctl output with very long urls (systemd bug)
+Defaults !syslog
+
 Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
 
@@ -27,3 +30,7 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
 
 Defaults!/home/yellowtent/box/src/scripts/collectlogs.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
+
+Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
+

setup/container/supervisor/conf.d/apphealthtask.conf

@@ -1,10 +0,0 @@
-[program:apphealthtask]
-command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/apphealthtask.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/box.conf

@@ -1,10 +0,0 @@
-[program:box]
-command=/usr/bin/node "/home/yellowtent/box/app.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/box.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/crashnotifier.conf

@@ -1,11 +0,0 @@
-[eventlistener:crashnotifier]
-command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
-events=PROCESS_STATE
-autostart=true
-autorestart=true
-redirect_stderr=false
-stderr_logfile=/var/log/supervisor/crashnotifier.log
-stderr_logfile_maxbytes=50MB
-stderr_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/janitor.conf

@@ -1,10 +0,0 @@
-[program:janitor]
-command=/usr/bin/node "/home/yellowtent/box/janitor.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/janitor.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/conf.d/oauthproxy.conf

@@ -1,10 +0,0 @@
-[program:oauthproxy]
-command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
-autostart=true
-autorestart=true
-redirect_stderr=true
-stdout_logfile=/var/log/supervisor/oauthproxy.log
-stdout_logfile_maxbytes=50MB
-stdout_logfile_backups=2
-user=yellowtent
-environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",BOX_ENV="cloudron",NODE_ENV="production"

setup/container/supervisor/supervisord.conf

@@ -1,33 +0,0 @@
-; supervisor config file
-
-; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
-[inet_http_server]
-port = 127.0.0.1:9001
-
-[supervisord]
-logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
-pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-logfile_maxbytes = 50MB
-logfile_backups=10
-loglevel = info
-nodaemon = false
-childlogdir = /var/log/supervisor/
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=http://127.0.0.1:9001
-
-; The [include] section can just contain the "files" setting.  This
-; setting can list multiple files (separated by whitespace or
-; newlines).  It can also contain wildcards.  The filenames are
-; interpreted as relative to this file.  Included files *cannot*
-; include files themselves.
-
-[include]
-files = conf.d/*.conf
-

setup/container/systemd/box.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Cloudron Admin
+OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
+; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
+BindsTo=systemd-journald.service
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+Restart=always
+ExecStart=/usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+; kill apptask processes as well
+KillMode=control-group
+User=yellowtent
+Group=yellowtent
+MemoryLimit=200M
+TimeoutStopSec=5s
+StartLimitInterval=1
+StartLimitBurst=60
+

setup/container/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smartserver
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service
+After=box.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/container/systemd/crashnotifier@.service

@@ -0,0 +1,15 @@
+# http://northernlightlabs.se/systemd.status.mail.on.unit.failure
+[Unit]
+Description=Cloudron Crash Notifier for %i
+# otherwise, systemd will kill this unit immediately as nobody requires it
+StopWhenUnneeded=false
+
+[Service]
+Type=idle
+WorkingDirectory=/home/yellowtent/box
+ExecStart="/home/yellowtent/box/crashnotifier.js" %I
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
+KillMode=process
+User=yellowtent
+Group=yellowtent
+MemoryLimit=50M

setup/splashpage.sh

@@ -29,10 +29,10 @@ infra_version="none"
 if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi
 
 echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"

setup/start.sh

@@ -38,7 +38,9 @@ set_progress "10" "Ensuring directories"
 # keep these in sync with paths.js
 [[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
 mkdir -p "${DATA_DIR}/box/appicons"
+mkdir -p "${DATA_DIR}/box/certs"
 mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/box/acme" # acme keys
 mkdir -p "${DATA_DIR}/graphite"
 
 mkdir -p "${DATA_DIR}/mysql"
@@ -47,6 +49,7 @@ mkdir -p "${DATA_DIR}/mongodb"
 mkdir -p "${DATA_DIR}/snapshots"
 mkdir -p "${DATA_DIR}/addons"
 mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+mkdir -p "${DATA_DIR}/acme" # acme challenges
 
 # bookkeep the version as part of data
 echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
@@ -89,31 +92,35 @@ EOF
 
 set_progress "28" "Setup collectd"
 cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
-# collectd 5.4.1 has some bug where we simply cannot get it to create df-vda1
-mkdir -p "${DATA_DIR}/graphite/whisper/collectd/localhost/"
-vda1_id=$(blkid -s UUID -o value /dev/vda1)
-ln -sfF "df-disk_by-uuid_${vda1_id}" "${DATA_DIR}/graphite/whisper/collectd/localhost/df-vda1"
 service collectd restart
 
 set_progress "30" "Setup nginx"
-# setup naked domain to use admin by default. app restoration will overwrite this config
 mkdir -p "${DATA_DIR}/nginx/applications"
+cp "${script_dir}/start/nginx/nginx.conf" "${DATA_DIR}/nginx/nginx.conf"
 cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"
 
-# generate the main nginx config file
-${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs" \
-    -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
-
 # generate these for update code paths as well to overwrite splash
+admin_cert_file="${DATA_DIR}/nginx/cert/host.cert"
+admin_key_file="${DATA_DIR}/nginx/cert/host.key"
+if [[ -f "${DATA_DIR}/box/certs/${admin_fqdn}.cert" && -f "${DATA_DIR}/box/certs/${admin_fqdn}.key" ]]; then
+    admin_cert_file="${DATA_DIR}/box/certs/${admin_fqdn}.cert"
+    admin_key_file="${DATA_DIR}/box/certs/${admin_fqdn}.key"
+fi
 ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
-    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\", \"certFilePath\": \"${admin_cert_file}\", \"keyFilePath\": \"${admin_key_file}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 
 mkdir -p "${DATA_DIR}/nginx/cert"
-echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
+if [[ -f "${DATA_DIR}/box/certs/host.cert" && -f "${DATA_DIR}/box/certs/host.key" ]]; then
-echo "${arg_tls_key}" > ${DATA_DIR}/nginx/cert/host.key
+    cp "${DATA_DIR}/box/certs/host.cert" "${DATA_DIR}/nginx/cert/host.cert"
+    cp "${DATA_DIR}/box/certs/host.key" "${DATA_DIR}/nginx/cert/host.key"
+else
+    echo "${arg_tls_cert}" > "${DATA_DIR}/nginx/cert/host.cert"
+    echo "${arg_tls_key}" > "${DATA_DIR}/nginx/cert/host.key"
+fi
 
 set_progress "33" "Changing ownership"
-chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}"
 
 set_progress "40" "Setting up infra"
 ${script_dir}/start/setup_infra.sh "${arg_fqdn}"
@@ -132,6 +139,7 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
     "adminEmail": "admin@${arg_fqdn}",
+    "provider": "${arg_provider}",
     "database": {
         "hostname": "localhost",
         "username": "root",
@@ -150,35 +158,55 @@ cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
 CONF_END
 EOF
 
+# Add Backup Configuration
+if [[ ! -z "${arg_backup_config}" ]]; then
+    echo "Add Backup Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
+fi
+
+# Add DNS Configuration
+if [[ ! -z "${arg_dns_config}" ]]; then
+    echo "Add DNS Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
+fi
+
+# Add Update Configuration
+if [[ ! -z "${arg_update_config}" ]]; then
+    echo "Add Update Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"update_config\", '$arg_update_config')" box
+fi
+
+# Add TLS Configuration
+if [[ ! -z "${arg_tls_config}" ]]; then
+    echo "Add TLS Config"
+
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
+fi
+
 # Add webadmin oauth client
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
 echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-echo "Add localhost test oauth cient"
+echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
 
-set_progress "80" "Reloading supervisor"
+set_progress "80" "Starting Cloudron"
-# looks like restarting supervisor completely is the only way to reload it
+systemctl start cloudron.target
-service supervisor stop || true
 
-echo -n "Waiting for supervisord to stop"
+sleep 2 # give systemd sometime to start the processes
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-
-echo "Starting supervisor"
-
-service supervisor start
-
-sleep 2 # give supervisor sometime to start the processes
 
 set_progress "85" "Reloading nginx"
 nginx -s reload

setup/start/collectd.conf

@@ -133,10 +133,10 @@ LoadPlugin nginx
 #   Globals true
 #</LoadPlugin>
 #LoadPlugin pinba
-LoadPlugin ping
+#LoadPlugin ping
 #LoadPlugin postgresql
 #LoadPlugin powerdns
-LoadPlugin processes
+#LoadPlugin processes
 #LoadPlugin protocols
 #<LoadPlugin python>
 #   Globals true
@@ -161,7 +161,7 @@ LoadPlugin tail
 #LoadPlugin users
 #LoadPlugin uuid
 #LoadPlugin varnish
-LoadPlugin vmem
+#LoadPlugin vmem
 #LoadPlugin vserver
 #LoadPlugin wireless
 LoadPlugin write_graphite
@@ -193,11 +193,11 @@ LoadPlugin write_graphite
 </Plugin>
 
 <Plugin df>
-   FSType "tmpfs"
+   FSType "ext4"
-   MountPoint "/dev"
+   FSType "btrfs"
 
    ReportByDevice true
-   IgnoreSelected true
+   IgnoreSelected false
 
    ValuesAbsolute true
    ValuesPercentage true
@@ -212,17 +212,6 @@ LoadPlugin write_graphite
    URL "http://127.0.0.1/nginx_status"
 </Plugin>
 
-<Plugin ping>
-   Host "google.com"
-   Interval 1.0
-   Timeout 0.9
-   TTL 255
-</Plugin>
-
-<Plugin processes>
-   ProcessMatch "app" "node app.js"
-</Plugin>
-
 <Plugin swap>
    ReportByDevice false
    ReportBytes true
@@ -255,10 +244,6 @@ LoadPlugin write_graphite
    </File>
 </Plugin>
 
-<Plugin vmem>
-   Verbose false
-</Plugin>
-
 <Plugin write_graphite>
    <Node "graphing">
        Host "localhost"

setup/start/nginx/appconfig.ejs

@@ -10,8 +10,8 @@ server {
 
     ssl                  on;
     # paths are relative to prefix and not to this file
-    ssl_certificate      cert/host.cert;
+    ssl_certificate      <%= certFilePath %>;
-    ssl_certificate_key  cert/host.key;
+    ssl_certificate_key  <%= keyFilePath %>;
     ssl_session_timeout  5m;
     ssl_session_cache shared:SSL:50m;
 
@@ -37,7 +37,8 @@ server {
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection $connection_upgrade;
 
-    error_page 500 502 503 504 @appstatus;
+    # only serve up the status page if we get proxy gateway errors
+    error_page 502 503 504 @appstatus;
     location @appstatus {
         return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
     }
@@ -57,19 +58,24 @@ server {
             client_max_body_size 1m;
         }
 
-        # graphite paths
+        location ~ ^/api/v1/apps/.*/exec$ {
-        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+            proxy_pass   http://127.0.0.1:3000;
-            proxy_pass   http://127.0.0.1:8000;
+            proxy_read_timeout 30m;
-            client_max_body_size 1m;
         }
 
+        # graphite paths
+        # location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+        #     proxy_pass   http://127.0.0.1:8000;
+        #     client_max_body_size 1m;
+        # }
+
         location / {
             root   <%= sourceDir %>/webadmin/dist;
             index  index.html index.htm;
         }
 
 <% } else if ( endpoint === 'oauthproxy' ) { %>
-        proxy_pass http://127.0.0.1:4000;
+        proxy_pass http://127.0.0.1:3003;
         proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
 <% } else if ( endpoint === 'app' ) { %>
         proxy_pass http://127.0.0.1:<%= port %>;

setup/start/nginx/nginx.conf

@@ -38,6 +38,12 @@ http {
             deny all;
         }
 
+        # acme challenges
+        location /.well-known/acme-challenge/ {
+            default_type text/plain;
+            alias /home/yellowtent/data/acme/;
+        }
+
         location / {
             # redirect everything to HTTPS
             return 301 https://$host$request_uri;
@@ -52,14 +58,31 @@ http {
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
+        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+        proxy_buffer_size       128k;
+        proxy_buffers           4 256k;
+        proxy_busy_buffers_size 256k;
+
+        # Disable check to allow unlimited body sizes
+        client_max_body_size 0;
+
         error_page 404 = @fallback;
         location @fallback {
             internal;
-            root <%= sourceDir %>/webadmin/dist;
+            root /home/yellowtent/box/webadmin/dist;
             rewrite ^/$ /nakeddomain.html break;
         }
 
-        return 404;
+        location / {
+            internal;
+            root /home/yellowtent/box/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
+        location /api/ {
+            proxy_pass http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
     }
 
     include applications/*.conf;

setup/start/setup_infra.sh

@@ -28,21 +28,32 @@ fi
 
 # graphite
 graphite_container_id=$(docker run --restart=always -d --name="graphite" \
+    -m 75m \
+    --memory-swap 150m \
     -p 127.0.0.1:2003:2003 \
     -p 127.0.0.1:2004:2004 \
     -p 127.0.0.1:8000:8000 \
     -v "${DATA_DIR}/graphite:/app/data" \
+    --read-only -v /tmp -v /run \
     "${GRAPHITE_IMAGE}")
 echo "Graphite container id: ${graphite_container_id}"
+if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old graphite images"
+fi
 
-# mail
+# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
 mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -p 127.0.0.1:25:25 \
+    -m 75m \
+    --memory-swap 150m \
     -h "${arg_fqdn}" \
     -e "DOMAIN_NAME=${arg_fqdn}" \
     -v "${DATA_DIR}/box/mail:/app/data" \
+    --read-only -v /tmp -v /run \
     "${MAIL_IMAGE}")
 echo "Mail container id: ${mail_container_id}"
+if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mail images"
+fi
 
 # mysql
 mysql_addon_root_password=$(pwgen -1 -s)
@@ -52,11 +63,17 @@ readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
 readonly MYSQL_ROOT_HOST='${docker0_ip}'
 EOF
 mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -m 256m \
+    --memory-swap 512m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mysql:/var/lib/mysql" \
     -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
     "${MYSQL_IMAGE}")
 echo "MySQL container id: ${mysql_container_id}"
+if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mysql images"
+fi
 
 # postgresql
 postgresql_addon_root_password=$(pwgen -1 -s)
@@ -64,11 +81,17 @@ cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
 readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
 EOF
 postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
     -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
     "${POSTGRESQL_IMAGE}")
 echo "PostgreSQL container id: ${postgresql_container_id}"
+if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old postgresql images"
+fi
 
 # mongodb
 mongodb_addon_root_password=$(pwgen -1 -s)
@@ -76,20 +99,31 @@ cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
 readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
 EOF
 mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -m 100m \
+    --memory-swap 200m \
     -h "${arg_fqdn}" \
     -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
     -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+    --read-only -v /tmp -v /run \
     "${MONGODB_IMAGE}")
 echo "Mongodb container id: ${mongodb_container_id}"
+if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old mongodb images"
+fi
 
+# redis
+if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
+    echo "Removed old redis images"
+fi
+
+# only touch apps in installed state. any other state is just resumed by the taskmanager
 if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new and restoring cloudons), download app backups
+    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
     echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore" WHERE installationState = "installed"' box
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
 else
     # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure" WHERE installationState = "installed"' box
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
 fi
 
 echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"
-

setup/stop.sh

@@ -2,14 +2,6 @@
 
 set -eu -o pipefail
 
-echo "Stopping box code"
+echo "Stopping cloudron"
-
-service supervisor stop || true
-
-echo -n "Waiting for supervisord to stop"
-while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
-    echo -n "."
-    sleep 1
-done
-echo ""
 
+systemctl stop cloudron.target

src/addons.js

@@ -9,10 +9,11 @@ exports = module.exports = {
     getEnvironment: getEnvironment,
     getLinksSync: getLinksSync,
     getBindsSync: getBindsSync,
+    getContainerNamesSync: getContainerNamesSync,
 
     // exported for testing
-    _allocateOAuthCredentials: allocateOAuthCredentials,
+    _setupOauth: setupOauth,
-    _removeOAuthCredentials: removeOAuthCredentials
+    _teardownOauth: teardownOauth
 };
 
 var appdb = require('./appdb.js'),
@@ -23,64 +24,36 @@ var appdb = require('./appdb.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:addons'),
-    docker = require('./docker.js'),
+    docker = require('./docker.js').connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
     MemoryStream = require('memorystream'),
     once = require('once'),
-    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     spawn = child_process.spawn,
-    tokendb = require('./tokendb.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
+    uuid = require('node-uuid');
-    vbox = require('./vbox.js'),
-    _ = require('underscore');
 
-var NOOP = function (app, callback) { return callback(); };
+var NOOP = function (app, options, callback) { return callback(); };
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
-    oauth: {
-        setup: allocateOAuthCredentials,
-        teardown: removeOAuthCredentials,
-        backup: NOOP,
-        restore: allocateOAuthCredentials
-    },
-    token: {
-        setup: allocateAccessToken,
-        teardown: removeAccessToken,
-        backup: NOOP,
-        restore: allocateAccessToken
-    },
     ldap: {
         setup: setupLdap,
         teardown: teardownLdap,
         backup: NOOP,
         restore: setupLdap
     },
-    sendmail: {
+    localstorage: {
-        setup: setupSendMail,
+        setup: NOOP, // docker creates the directory for us
-        teardown: teardownSendMail,
+        teardown: NOOP,
-        backup: NOOP,
+        backup: NOOP, // no backup because it's already inside app data
-        restore: setupSendMail
+        restore: NOOP
-    },
-    mysql: {
-        setup: setupMySql,
-        teardown: teardownMySql,
-        backup: backupMySql,
-        restore: restoreMySql,
-    },
-    postgresql: {
-        setup: setupPostgreSql,
-        teardown: teardownPostgreSql,
-        backup: backupPostgreSql,
-        restore: restorePostgreSql
     },
     mongodb: {
         setup: setupMongoDb,
@@ -88,18 +61,48 @@ var KNOWN_ADDONS = {
         backup: backupMongoDb,
         restore: restoreMongoDb
     },
+    mysql: {
+        setup: setupMySql,
+        teardown: teardownMySql,
+        backup: backupMySql,
+        restore: restoreMySql,
+    },
+    oauth: {
+        setup: setupOauth,
+        teardown: teardownOauth,
+        backup: NOOP,
+        restore: setupOauth
+    },
+    postgresql: {
+        setup: setupPostgreSql,
+        teardown: teardownPostgreSql,
+        backup: backupPostgreSql,
+        restore: restorePostgreSql
+    },
     redis: {
         setup: setupRedis,
         teardown: teardownRedis,
-        backup: NOOP, // no backup because we store redis as part of app's volume
+        backup: backupRedis,
         restore: setupRedis // same thing
     },
-    localstorage: {
+    sendmail: {
-        setup: NOOP, // docker creates the directory for us
+        setup: setupSendMail,
+        teardown: teardownSendMail,
+        backup: NOOP,
+        restore: setupSendMail
+    },
+    scheduler: {
+        setup: NOOP,
         teardown: NOOP,
-        backup: NOOP, // no backup because it's already inside app data
+        backup: NOOP,
         restore: NOOP
     },
+    simpleauth: {
+        setup: setupSimpleAuth,
+        teardown: teardownSimpleAuth,
+        backup: NOOP,
+        restore: setupSimpleAuth
+    },
     _docker: {
         setup: NOOP,
         teardown: NOOP,
@@ -129,9 +132,9 @@ function setupAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        debugApp(app, 'Setting up addon %s', addon);
+        debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]);
 
-        KNOWN_ADDONS[addon].setup(app, iteratorCallback);
+        KNOWN_ADDONS[addon].setup(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -147,9 +150,9 @@ function teardownAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        debugApp(app, 'Tearing down addon %s', addon);
+        debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]);
 
-        KNOWN_ADDONS[addon].teardown(app, iteratorCallback);
+        KNOWN_ADDONS[addon].teardown(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -167,7 +170,7 @@ function backupAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        KNOWN_ADDONS[addon].backup(app, iteratorCallback);
+        KNOWN_ADDONS[addon].backup(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -185,7 +188,7 @@ function restoreAddons(app, addons, callback) {
     async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) {
         if (!(addon in KNOWN_ADDONS)) return iteratorCallback(new Error('No such addon:' + addon));
 
-        KNOWN_ADDONS[addon].restore(app, iteratorCallback);
+        KNOWN_ADDONS[addon].restore(app, addons[addon], iteratorCallback);
     }, callback);
 }
 
@@ -237,22 +240,44 @@ function getBindsSync(app, addons) {
     return binds;
 }
 
-function allocateOAuthCredentials(app, callback) {
+function getContainerNamesSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
+    assert(!addons || typeof addons === 'object');
+
+    var names = [ ];
+
+    if (!addons) return names;
+
+    for (var addon in addons) {
+        switch (addon) {
+        case 'scheduler':
+            // names here depend on how scheduler.js creates containers
+            names = names.concat(Object.keys(addons.scheduler).map(function (taskName) { return app.id + '-' + taskName; }));
+            break;
+        default: break;
+        }
+    }
+
+    return names;
+}
+
+function setupOauth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-addon-' + uuid.v4();
+    var id = 'cid-' + uuid.v4();
     var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
-    var scope = 'profile,roleUser';
+    var scope = 'profile';
 
-    debugApp(app, 'allocateOAuthCredentials: id:%s clientSecret:%s', id, clientSecret);
+    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
 
-    clientdb.delByAppId('addon-' + appId, function (error) { // remove existing creds
+    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
         if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
 
-        clientdb.add(id, 'addon-' + appId, clientSecret, redirectURI, scope, function (error) {
+        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
             if (error) return callback(error);
 
             var env = [
@@ -268,29 +293,79 @@ function allocateOAuthCredentials(app, callback) {
     });
 }
 
-function removeOAuthCredentials(app, callback) {
+function teardownOauth(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debugApp(app, 'removeOAuthCredentials');
+    debugApp(app, 'teardownOauth');
 
-    clientdb.delByAppId('addon-' + app.id, function (error) {
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
 }
 
-function setupLdap(app, callback) {
+function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var appId = app.id;
+    var id = 'cid-' + uuid.v4();
+    var scope = 'profile';
+
+    debugApp(app, 'setupSimpleAuth: id:%s', id);
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
+
+        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+            if (error) return callback(error);
+
+            var env = [
+                'SIMPLE_AUTH_SERVER=172.17.0.1',
+                'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + id
+            ];
+
+            debugApp(app, 'Setting simple auth addon config to %j', env);
+
+            appdb.setAddonConfig(appId, 'simpleauth', env, callback);
+        });
+    });
+}
+
+function teardownSimpleAuth(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'teardownSimpleAuth');
+
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+
+        appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
+    });
+}
+
+function setupLdap(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'LDAP_SERVER=172.17.42.1',
+        'LDAP_SERVER=172.17.0.1',
-        'LDAP_PORT=3002',
+        'LDAP_PORT=' + config.get('ldapPort'),
-        'LDAP_URL=ldap://172.17.42.1:3002',
+        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
-        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron'
+        'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
+        'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
+        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
     ];
 
     debugApp(app, 'Setting up LDAP');
@@ -298,8 +373,9 @@ function setupLdap(app, callback) {
     appdb.setAddonConfig(app.id, 'ldap', env, callback);
 }
 
-function teardownLdap(app, callback) {
+function teardownLdap(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Tearing down LDAP');
@@ -307,14 +383,18 @@ function teardownLdap(app, callback) {
     appdb.unsetAddonConfig(app.id, 'ldap', callback);
 }
 
-function setupSendMail(app, callback) {
+function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    var username = app.location ? app.location + '-app' : 'no-reply'; // use no-reply for bare domains
+
     var env = [
         'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=25',
+        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + (app.location || app.id), // use app.id for bare domains
+        'MAIL_SMTP_USERNAME=' + username,
+        'MAIL_SMTP_PASSWORD=' + hat(256), // this is ignored
         'MAIL_DOMAIN=' + config.fqdn()
     ];
 
@@ -323,8 +403,9 @@ function setupSendMail(app, callback) {
     appdb.setAddonConfig(app.id, 'sendmail', env, callback);
 }
 
-function teardownSendMail(app, callback) {
+function teardownSendMail(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Tearing down sendmail');
@@ -332,14 +413,15 @@ function teardownSendMail(app, callback) {
     appdb.unsetAddonConfig(app.id, 'sendmail', callback);
 }
 
-function setupMySql(app, callback) {
+function setupMySql(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Setting up mysql');
 
     var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'add', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'add-prefix' : 'add', app.id ];
 
     container.exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true }, function (error, execContainer) {
         if (error) return callback(error);
@@ -366,9 +448,13 @@ function setupMySql(app, callback) {
     });
 }
 
-function teardownMySql(app, callback) {
+function teardownMySql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var container = docker.getContainer('mysql');
-    var cmd = [ '/addons/mysql/service.sh', 'remove', app.id ];
+    var cmd = [ '/addons/mysql/service.sh', options.multipleDatabases ? 'remove-prefix' : 'remove', app.id ];
 
     debugApp(app, 'Tearing down mysql');
 
@@ -388,7 +474,7 @@ function teardownMySql(app, callback) {
     });
 }
 
-function backupMySql(app, callback) {
+function backupMySql(app, options, callback) {
     debugApp(app, 'Backing up mysql');
 
     callback = once(callback); // ChildProcess exit may or may not be called after error
@@ -396,7 +482,7 @@ function backupMySql(app, callback) {
     var output = fs.createWriteStream(path.join(paths.DATA_DIR, app.id, 'mysqldump'));
     output.on('error', callback);
 
-    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', 'backup', app.id ]);
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'backup-prefix' : 'backup', app.id ]);
     cp.on('error', callback);
     cp.on('exit', function (code, signal) {
         debugApp(app, 'backupMySql: done. code:%s signal:%s', code, signal);
@@ -407,10 +493,10 @@ function backupMySql(app, callback) {
     cp.stderr.pipe(process.stderr);
 }
 
-function restoreMySql(app, callback) {
+function restoreMySql(app, options, callback) {
     callback = once(callback); // ChildProcess exit may or may not be called after error
 
-    setupMySql(app, function (error) {
+    setupMySql(app, options, function (error) {
         if (error) return callback(error);
 
         debugApp(app, 'restoreMySql');
@@ -419,7 +505,7 @@ function restoreMySql(app, callback) {
         input.on('error', callback);
 
         // cannot get this to work through docker.exec
-        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', 'restore', app.id ]);
+        var cp = spawn('/usr/bin/docker', [ 'exec', '-i', 'mysql', '/addons/mysql/service.sh', options.multipleDatabases ? 'restore-prefix' : 'restore', app.id ]);
         cp.on('error', callback);
         cp.on('exit', function (code, signal) {
             debugApp(app, 'restoreMySql: done %s %s', code, signal);
@@ -432,8 +518,9 @@ function restoreMySql(app, callback) {
     });
 }
 
-function setupPostgreSql(app, callback) {
+function setupPostgreSql(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Setting up postgresql');
@@ -466,7 +553,11 @@ function setupPostgreSql(app, callback) {
     });
 }
 
-function teardownPostgreSql(app, callback) {
+function teardownPostgreSql(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var container = docker.getContainer('postgresql');
     var cmd = [ '/addons/postgresql/service.sh', 'remove', app.id ];
 
@@ -488,7 +579,7 @@ function teardownPostgreSql(app, callback) {
     });
 }
 
-function backupPostgreSql(app, callback) {
+function backupPostgreSql(app, options, callback) {
     debugApp(app, 'Backing up postgresql');
 
     callback = once(callback); // ChildProcess exit may or may not be called after error
@@ -507,10 +598,10 @@ function backupPostgreSql(app, callback) {
     cp.stderr.pipe(process.stderr);
 }
 
-function restorePostgreSql(app, callback) {
+function restorePostgreSql(app, options, callback) {
     callback = once(callback); // ChildProcess exit may or may not be called after error
 
-    setupPostgreSql(app, function (error) {
+    setupPostgreSql(app, options, function (error) {
         if (error) return callback(error);
 
         debugApp(app, 'restorePostgreSql');
@@ -532,8 +623,9 @@ function restorePostgreSql(app, callback) {
     });
 }
 
-function setupMongoDb(app, callback) {
+function setupMongoDb(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debugApp(app, 'Setting up mongodb');
@@ -566,7 +658,11 @@ function setupMongoDb(app, callback) {
     });
 }
 
-function teardownMongoDb(app, callback) {
+function teardownMongoDb(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var container = docker.getContainer('mongodb');
     var cmd = [ '/addons/mongodb/service.sh', 'remove', app.id ];
 
@@ -588,7 +684,7 @@ function teardownMongoDb(app, callback) {
     });
 }
 
-function backupMongoDb(app, callback) {
+function backupMongoDb(app, options, callback) {
     debugApp(app, 'Backing up mongodb');
 
     callback = once(callback); // ChildProcess exit may or may not be called after error
@@ -607,10 +703,10 @@ function backupMongoDb(app, callback) {
     cp.stderr.pipe(process.stderr);
 }
 
-function restoreMongoDb(app, callback) {
+function restoreMongoDb(app, options, callback) {
     callback = once(callback); // ChildProcess exit may or may not be called after error
 
-    setupMongoDb(app, function (error) {
+    setupMongoDb(app, options, function (error) {
         if (error) return callback(error);
 
         debugApp(app, 'restoreMongoDb');
@@ -643,15 +739,35 @@ function forwardRedisPort(appId, callback) {
         var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
         if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
 
-        vbox.forwardFromHostToVirtualBox('redis-' + appId, redisPort);
-
         return callback(null);
     });
 }
 
+function stopAndRemoveRedis(container, callback) {
+    function ignoreError(func) {
+        return function (callback) {
+            func(function (error) {
+                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
+                callback();
+            });
+        };
+    }
+
+    // stopping redis with SIGTERM makes it commit the database to disk
+    async.series([
+        ignoreError(container.stop.bind(container, { t: 10 })),
+        ignoreError(container.wait.bind(container)),
+        ignoreError(container.remove.bind(container, { force: true, v: true }))
+    ], callback);
+}
+
 // Ensures that app's addon redis container is running. Can be called when named container already exists/running
-function setupRedis(app, callback) {
+function setupRedis(app, options, callback) {
-    var redisPassword = generatePassword(64, false /* memorable */);
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var redisPassword = generatePassword(64, false /* memorable */, /[\w\d_]/); // ensure no / in password for being sed friendly (and be uri friendly)
     var redisVarsFile = path.join(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
     var redisDataDir = path.join(paths.DATA_DIR, app.id + '/redis');
 
@@ -663,30 +779,31 @@ function setupRedis(app, callback) {
 
     var createOptions = {
         name: 'redis-' + app.id,
-        Hostname: config.appFqdn(app.location),
+        Hostname: 'redis-' + app.location,
         Tty: true,
-        Image: 'cloudron/redis:0.3.1',
+        Image: 'cloudron/redis:0.8.0', // if you change this, fix setup/INFRA_VERSION as well
         Cmd: null,
-        Volumes: {},
+        Volumes: {
-        VolumesFrom: []
+            '/tmp': {},
-    };
+            '/run': {}
-
+        },
-    var isMac = os.platform() === 'darwin';
+        VolumesFrom: [],
-
+        HostConfig: {
-    var startOptions = {
             Binds: [
                 redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
                 redisDataDir + ':/var/lib/redis:rw'
             ],
-        // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work
+            Memory: 1024 * 1024 * 75, // 100mb
-        // On linux, export to localhost only for testing purposes and not for the app itself
+            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
             PortBindings: {
-            '6379/tcp': [{ HostPort: '0', HostIp: isMac ? '0.0.0.0' : '127.0.0.1' }]
+                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
             },
+            ReadonlyRootfs: true,
             RestartPolicy: {
                 'Name': 'always',
                 'MaximumRetryCount': 0
             }
+        }
     };
 
     var env = [
@@ -697,11 +814,11 @@ function setupRedis(app, callback) {
     ];
 
     var redisContainer = docker.getContainer(createOptions.name);
-    redisContainer.remove({ force: true, v: false }, function (ignoredError) {
+    stopAndRemoveRedis(redisContainer, function () {
         docker.createContainer(createOptions, function (error) {
             if (error && error.statusCode !== 409) return callback(error); // if not already created
 
-            redisContainer.start(startOptions, function (error) {
+            redisContainer.start(function (error) {
                 if (error && error.statusCode !== 304) return callback(error); // if not already running
 
                 appdb.setAddonConfig(app.id, 'redis', env, function (error) {
@@ -714,19 +831,21 @@ function setupRedis(app, callback) {
     });
 }
 
-function teardownRedis(app, callback) {
+function teardownRedis(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
    var container = docker.getContainer('redis-' + app.id);
 
    var removeOptions = {
        force: true, // kill container if it's running
-       v: false // removes volumes associated with the container
+       v: true // removes volumes associated with the container
    };
 
    container.remove(removeOptions, function (error) {
        if (error && error.statusCode !== 404) return callback(new Error('Error removing container:' + error));
 
-       vbox.unforwardFromHostToVirtualBox('redis-' + app.id);
-
        safe.fs.unlinkSync(paths.ADDON_CONFIG_DIR, 'redis-' + app.id + '_vars.sh');
 
         shell.sudo('teardownRedis', [ RMAPPDIR_CMD, app.id + '/redis' ], function (error, stdout, stderr) {
@@ -737,40 +856,18 @@ function teardownRedis(app, callback) {
    });
 }
 
-function allocateAccessToken(app, callback) {
+function backupRedis(app, options, callback) {
-    assert.strictEqual(typeof app, 'object');
+    debugApp(app, 'Backing up redis');
-    assert.strictEqual(typeof callback, 'function');
 
-    var token = tokendb.generateToken();
+    callback = once(callback); // ChildProcess exit may or may not be called after error
-    var expiresAt = Number.MAX_SAFE_INTEGER;    // basically never expire
-    var scopes = 'profile,users';               // TODO This should be put into the manifest and the user should know those
-    var clientId = '';                          // meaningless for apps so far
 
-   tokendb.delByIdentifier(tokendb.PREFIX_APP + app.id, function (error) {
+    var cp = spawn('/usr/bin/docker', [ 'exec', 'redis-' + app.id, '/addons/redis/service.sh', 'backup' ]);
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
+    cp.on('error', callback);
-
+    cp.on('exit', function (code, signal) {
-        tokendb.add(token, tokendb.PREFIX_APP + app.id, clientId, expiresAt, scopes, function (error) {
+        debugApp(app, 'backupRedis: done. code:%s signal:%s', code, signal);
-            if (error) return callback(error);
+        if (!callback.called) callback(code ? 'backupRedis failed with status ' + code : null);
-
-            var env = [
-                'CLOUDRON_TOKEN=' + token
-            ];
-
-            debugApp(app, 'Setting token addon config to %j', env);
-
-            appdb.setAddonConfig(appId, 'token', env, callback);
-        });
     });
+
+    cp.stdout.pipe(process.stdout);
+    cp.stderr.pipe(process.stderr);
 }
-
-function removeAccessToken(app, callback) {
-    assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    tokendb.delByIdentifier(tokendb.PREFIX_APP + app.id, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
-
-        appdb.unsetAddonConfig(app.id, 'token', callback);
-    });
-}
-

src/appdb.js

@@ -6,6 +6,7 @@ exports = module.exports = {
     get: get,
     getBySubdomain: getBySubdomain,
     getByHttpPort: getByHttpPort,
+    getByContainerId: getByContainerId,
     add: add,
     exists: exists,
     del: del,
@@ -35,12 +36,12 @@ exports = module.exports = {
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
-    // run codes (keep in sync in UI)
     RSTATE_RUNNING: 'running',
     RSTATE_PENDING_START: 'pending_start',
     RSTATE_PENDING_STOP: 'pending_stop',
     RSTATE_STOPPED: 'stopped', // app stopped by use
 
+    // run codes (keep in sync in UI)
     HEALTH_HEALTHY: 'healthy',
     HEALTH_UNHEALTHY: 'unhealthy',
     HEALTH_ERROR: 'error',
@@ -56,13 +57,9 @@ var assert = require('assert'),
     safe = require('safetydance'),
     util = require('util');
 
-var APPS_FIELDS = [ 'id', 'appStoreId', 'installationState', 'installationProgress', 'runState',
-    'health', 'containerId', 'manifestJson', 'httpPort', 'location', 'dnsRecordId',
-    'accessRestriction', 'lastBackupId', 'lastBackupConfigJson', 'oldConfigJson' ].join(',');
-
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestriction', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -94,6 +91,11 @@ function postProcess(result) {
     for (var i = 0; i < environmentVariables.length; i++) {
         result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
     }
+
+    assert(result.accessRestrictionJson === null || typeof result.accessRestrictionJson === 'string');
+    result.accessRestriction = safe.JSON.parse(result.accessRestrictionJson);
+    if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+    delete result.accessRestrictionJson;
 }
 
 function get(id, callback) {
@@ -144,6 +146,22 @@ function getByHttpPort(httpPort, callback) {
     });
 }
 
+function getByContainerId(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + APPS_FIELDS_PREFIXED + ','
+        + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables'
+        + '  FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId WHERE containerId = ? GROUP BY apps.id', [ containerId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -159,24 +177,26 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, callback) {
+function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert.strictEqual(typeof memoryLimit, 'number');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
     var manifestJson = JSON.stringify(manifest);
+    var accessRestrictionJson = JSON.stringify(accessRestriction);
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestriction) VALUES (?, ?, ?, ?, ?, ?)',
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit) VALUES (?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestriction ]
+        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -261,6 +281,7 @@ function updateWithConstraints(id, app, constraints, callback) {
     assert.strictEqual(typeof constraints, 'string');
     assert.strictEqual(typeof callback, 'function');
     assert(!('portBindings' in app) || typeof app.portBindings === 'object');
+    assert(!('accessRestriction' in app) || typeof app.accessRestriction === 'object' || app.accessRestriction === '');
 
     var queries = [ ];
 
@@ -285,6 +306,9 @@ function updateWithConstraints(id, app, constraints, callback) {
         } else if (p === 'oldConfig') {
             fields.push('oldConfigJson = ?');
             values.push(JSON.stringify(app[p]));
+        } else if (p === 'accessRestriction') {
+            fields.push('accessRestrictionJson = ?');
+            values.push(JSON.stringify(app[p]));
         } else if (p !== 'portBindings') {
             fields.push(p + ' = ?');
             values.push(app[p]);
@@ -335,6 +359,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
 
     // Rules are:
     // uninstall is allowed in any state
+    // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
     // update and configure are allowed only in installed state
 
@@ -342,7 +367,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
         updateWithConstraints(appId, values, '', callback);
     } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
         updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_CONFIGURE || installationState === exports.ISTATE_PENDING_BACKUP) {
         updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
     } else {
         callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));

src/apphealthmonitor.js

@@ -1,42 +1,35 @@
-#!/usr/bin/env node
-
 'use strict';
 
-require('supererror')({ splatchError: true });
+var appdb = require('./appdb.js'),
-
-var appdb = require('./src/appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    database = require('./src/database.js'),
+    config = require('./config.js'),
-    DatabaseError = require('./src/databaseerror.js'),
+    DatabaseError = require('./databaseerror.js'),
-    debug = require('debug')('box:apphealthtask'),
+    debug = require('debug')('box:apphealthmonitor'),
-    docker = require('./src/docker.js'),
+    docker = require('./docker.js').connection,
-    mailer = require('./src/mailer.js'),
+    mailer = require('./mailer.js'),
     superagent = require('superagent'),
     util = require('util');
 
 exports = module.exports = {
-    run: run
+    start: start,
+    stop: stop
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
 var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
 var gHealthInfo = { }; // { time, emailSent }
+var gRunTimeout = null;
+var gDockerEventStream = null;
 
-function debugApp(app, args) {
+function debugApp(app) {
     assert(!app || typeof app === 'object');
 
-    var prefix = app ? app.location : '(no app)';
+    var prefix = app ? (app.location || 'naked_domain') : '(no app)';
-    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+    var manifestAppId = app ? app.manifest.id : '';
-}
+    var id = app ? app.id : '';
 
-function initialize(callback) {
+    debug(prefix + ' ' + manifestAppId + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)) + ' - ' + id);
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        database.initialize,
-        mailer.initialize
-    ], callback);
 }
 
 function setHealth(app, health, callback) {
@@ -57,7 +50,7 @@ function setHealth(app, health, callback) {
 
         debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000));
 
-        mailer.appDied(app);
+        if (app.appStoreId !== '') mailer.appDied(app); // do not send mails for dev apps
         gHealthInfo[app.id].emailSent = true;
     } else {
         debugApp(app, 'waiting for sometime to update the app health');
@@ -100,15 +93,17 @@ function checkAppHealth(app, callback) {
         var healthCheckUrl = 'http://127.0.0.1:' + app.httpPort + manifest.healthCheckPath;
         superagent
             .get(healthCheckUrl)
+            .set('Host', config.appFqdn(app.location)) // required for some apache configs with rewrite rules
             .redirects(0)
             .timeout(HEALTHCHECK_INTERVAL)
             .end(function (error, res) {
-
+            if (error && !error.response) {
-            if (error || res.status >= 400) { // 2xx and 3xx are ok
+                debugApp(app, 'not alive (network error): %s', error.message);
+                setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
+            } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
                 debugApp(app, 'not alive : %s', error || res.status);
                 setHealth(app, appdb.HEALTH_UNHEALTHY, callback);
             } else {
-                debugApp(app, 'alive');
                 setHealth(app, appdb.HEALTH_HEALTHY, callback);
             }
         });
@@ -121,6 +116,13 @@ function processApps(callback) {
 
         async.each(apps, checkAppHealth, function (error) {
             if (error) console.error(error);
+
+            var alive = apps
+                .filter(function (a) { return a.installationState === appdb.ISTATE_INSTALLED && a.runState === appdb.RSTATE_RUNNING && a.health === appdb.HEALTH_HEALTHY; })
+                .map(function (a) { return (a.location || 'naked_domain') + '|' + a.manifest.id; }).join(', ');
+
+            debug('apps alive: [%s]', alive);
+
             callback(null);
         });
     });
@@ -130,18 +132,69 @@ function run() {
     processApps(function (error) {
         if (error) console.error(error);
 
-        setTimeout(run, HEALTHCHECK_INTERVAL);
+        gRunTimeout = setTimeout(run, HEALTHCHECK_INTERVAL);
     });
 }
 
-if (require.main === module) {
+/*
-    initialize(function (error) {
+    OOM can be tested using stress tool like so:
-        if (error) {
+        docker run -ti -m 100M cloudron/base:0.3.3 /bin/bash
-            console.error('apphealth task exiting with error', error);
+        apt-get update && apt-get install stress
-            process.exit(1);
+        stress --vm 1 --vm-bytes 200M --vm-hang 0
-        }
+*/
+function processDockerEvents() {
+    // note that for some reason, the callback is called only on the first event
+    debug('Listening for docker events');
+    docker.getEvents({ filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) {
+        if (error) return console.error(error);
+
+        gDockerEventStream = stream;
+
+        stream.setEncoding('utf8');
+        stream.on('data', function (data) {
+            var ev = JSON.parse(data);
+            debug('Container ' + ev.id + ' went OOM');
+            appdb.getByContainerId(ev.id, function (error, app) {
+                var program = error || !app.appStoreId ? ev.id : app.appStoreId;
+                var context = JSON.stringify(ev);
+                if (app) context = context + '\n\n' + JSON.stringify(app, null, 4) + '\n';
+
+                debug('OOM Context: %s', context);
+
+                // do not send mails for dev apps
+                if (error || app.appStoreId !== '') mailer.sendCrashNotification(program, context); // app can be null if it's an addon crash
+            });
+        });
+
+        stream.on('error', function (error) {
+            console.error('Error reading docker events', error);
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+
+        stream.on('end', function () {
+            console.error('Docker event stream ended');
+            gDockerEventStream = null; // TODO: reconnect?
+        });
+    });
+}
+
+function start(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Starting apphealthmonitor');
+
+    processDockerEvents();
 
     run();
-    });
+
+    callback();
 }
 
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    clearTimeout(gRunTimeout);
+    gDockerEventStream.end();
+
+    callback();
+}

src/apps.js

@@ -5,9 +5,13 @@
 exports = module.exports = {
     AppsError: AppsError,
 
+    hasAccessTo: hasAccessTo,
+
     get: get,
     getBySubdomain: getBySubdomain,
+    getByIpAddress: getByIpAddress,
     getAll: getAll,
+    getAllByUser: getAllByUser,
     purchase: purchase,
     install: install,
     configure: configure,
@@ -20,8 +24,8 @@ exports = module.exports = {
 
     backup: backup,
     backupApp: backupApp,
+    listBackups: listBackups,
 
-    getLogStream: getLogStream,
     getLogs: getLogs,
 
     start: start,
@@ -37,7 +41,8 @@ exports = module.exports = {
 
     // exported for testing
     _validateHostname: validateHostname,
-    _validatePortBindings: validatePortBindings
+    _validatePortBindings: validatePortBindings,
+    _validateAccessRestriction: validateAccessRestriction
 };
 
 var addons = require('./addons.js'),
@@ -46,18 +51,21 @@ var addons = require('./addons.js'),
     async = require('async'),
     backups = require('./backups.js'),
     BackupsError = require('./backups.js').BackupsError,
+    certificates = require('./certificates.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apps'),
     docker = require('./docker.js'),
     fs = require('fs'),
+    groups = require('./groups.js'),
     manifestFormat = require('cloudron-manifestformat'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     shell = require('./shell.js'),
+    spawn = require('child_process').spawn,
     split = require('split'),
     superagent = require('superagent'),
     taskmanager = require('./taskmanager.js'),
@@ -114,6 +122,9 @@ AppsError.BAD_STATE = 'Bad State';
 AppsError.PORT_RESERVED = 'Port Reserved';
 AppsError.PORT_CONFLICT = 'Port Conflict';
 AppsError.BILLING_REQUIRED = 'Billing Required';
+AppsError.ACCESS_DENIED = 'Access denied';
+AppsError.USER_REQUIRED = 'User required';
+AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 
 // Hostname validation comes from RFC 1123 (section 2.1)
 // Domain name validation comes from RFC 2181 (Name syntax)
@@ -140,16 +151,19 @@ function validatePortBindings(portBindings, tcpPorts) {
     // these ports are reserved even if we listen only on 127.0.0.1 because we setup HostIp to be 127.0.0.1
     // for custom tcp ports
     var RESERVED_PORTS = [
-        22, /* ssh */
         25, /* smtp */
         53, /* dns */
         80, /* http */
         443, /* https */
+        919, /* ssh */
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
         2020, /* install server */
         config.get('port'), /* app server (lo) */
         config.get('internalPort'), /* internal app server (lo) */
+        config.get('ldapPort'), /* ldap server (lo) */
+        config.get('oauthProxyPort'), /* oauth proxy server (lo) */
+        config.get('simpleAuthPort'), /* simple auth server (lo) */
         3306, /* mysql (lo) */
         8000 /* graphite (lo) */
     ];
@@ -163,7 +177,7 @@ function validatePortBindings(portBindings, tcpPorts) {
         if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
         if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
 
-        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, + portBindings[env]);
+        if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, String(portBindings[env]));
     }
 
     // it is OK if there is no 1-1 mapping between values in manifest.tcpPorts and portBindings. missing values implies
@@ -176,6 +190,47 @@ function validatePortBindings(portBindings, tcpPorts) {
     return null;
 }
 
+function validateAccessRestriction(accessRestriction) {
+    assert.strictEqual(typeof accessRestriction, 'object');
+
+    if (accessRestriction === null) return null;
+
+    var noUsers = true, noGroups = true;
+
+    if (accessRestriction.users) {
+        if (!Array.isArray(accessRestriction.users)) return new Error('users array property required');
+        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+        noUsers = accessRestriction.users.length === 0;
+    }
+
+    if (accessRestriction.groups) {
+        if (!Array.isArray(accessRestriction.groups)) return new Error('groups array property required');
+        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new Error('All groups have to be strings');
+        noGroups = accessRestriction.groups.length === 0;
+    }
+
+    if (noUsers && noGroups) return new Error('users and groups array cannot both be empty');
+
+    return null;
+}
+
+function validateMemoryLimit(manifest, memoryLimit) {
+    assert.strictEqual(typeof manifest, 'object');
+    assert.strictEqual(typeof memoryLimit, 'number');
+
+    var min = manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+    var max = (4096 * 1024 * 1024);
+
+    // allow 0, which indicates that it is not set, the one from the manifest will be choosen but we don't commit any user value
+    // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
+    if (memoryLimit === 0) return null;
+
+    if (memoryLimit < min) return new Error('memoryLimit too small');
+    if (memoryLimit > max) return new Error('memoryLimit too large');
+
+    return null;
+}
+
 function getDuplicateErrorDetails(location, portBindings, error) {
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
@@ -203,6 +258,28 @@ function getIconUrlSync(app) {
     return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
 }
 
+function hasAccessTo(app, user, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof user, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (app.accessRestriction === null) return callback(null, true);
+
+    // check user access
+    if (app.accessRestriction.users.some(function (e) { return e === user.id; })) return callback(null, true);
+
+    // check group access
+    if (!app.accessRestriction.groups) return callback(null, false);
+
+    async.some(app.accessRestriction.groups, function (groupId, iteratorDone) {
+        groups.isMember(groupId, user.id, function (error, member) {
+            iteratorDone(!error && member); // async.some does not take error argument in callback
+        });
+    }, function (result) {
+        callback(null, result);
+    });
+}
+
 function get(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -233,6 +310,25 @@ function getBySubdomain(subdomain, callback) {
     });
 }
 
+function getByIpAddress(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.getContainerIdByIp(ip, function (error, containerId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        appdb.getByContainerId(containerId, function (error, app) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            app.iconUrl = getIconUrlSync(app);
+            app.fqdn = config.appFqdn(app.location);
+
+            callback(null, app);
+        });
+    });
+}
+
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -248,16 +344,19 @@ function getAll(callback) {
     });
 }
 
-function validateAccessRestriction(accessRestriction) {
+function getAllByUser(user, callback) {
-    // TODO: make the values below enumerations in the oauth code
+    assert.strictEqual(typeof user, 'object');
-    switch (accessRestriction) {
+    assert.strictEqual(typeof callback, 'function');
-    case '':
+
-    case 'roleUser':
+    getAll(function (error, result) {
-    case 'roleAdmin':
+        if (error) return callback(error);
-        return null;
+
-    default:
+        async.filter(result, function (app, callback) {
-        return new Error('Invalid accessRestriction');
+            hasAccessTo(app, user, function (error, hasAccess) {
-    }
+                callback(hasAccess);
+            });
+        }, callback.bind(null, null));  // never error
+    });
 }
 
 function purchase(appStoreId, callback) {
@@ -267,25 +366,32 @@ function purchase(appStoreId, callback) {
     // Skip purchase if appStoreId is empty
     if (appStoreId === '') return callback(null);
 
+    // Skip if we don't have an appstore token
+    if (config.token() === '') return callback(null);
+
     var url = config.apiServerOrigin() + '/api/v1/apps/' + appStoreId + '/purchase';
 
     superagent.post(url).query({ token: config.token() }).end(function (error, res) {
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error));
-        if (res.status === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
+        if (res.statusCode === 402) return callback(new AppsError(AppsError.BILLING_REQUIRED));
-        if (res.status !== 201 && res.status !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
+        if (res.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (res.statusCode !== 201 && res.statusCode !== 200) return callback(new Error(util.format('App purchase failed. %s %j', res.status, res.body)));
 
         callback(null);
     });
 }
 
-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, callback) {
+function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof accessRestriction, 'object');
     assert(!icon || typeof icon === 'string');
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof memoryLimit, 'number');
     assert.strictEqual(typeof callback, 'function');
 
     var error = manifestFormat.parse(manifest);
@@ -303,6 +409,16 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     error = validateAccessRestriction(accessRestriction);
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+    error = validateMemoryLimit(manifest, memoryLimit);
+    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+    // memoryLimit might come in as 0 if not specified
+    memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
+    // singleUser mode requires accessRestriction to contain exactly one user
+    if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
+    if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
+
     if (icon) {
         if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
 
@@ -311,15 +427,24 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
         }
     }
 
+    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
+    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
     debug('Will install app with id : ' + appId);
 
     purchase(appStoreId, function (error) {
         if (error) return callback(error);
 
-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, function (error) {
+        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, function (error) {
             if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
+            // save cert to data/box/certs
+            if (cert && key) {
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+            }
+
             taskmanager.restartAppTask(appId);
 
             callback(null);
@@ -327,11 +452,14 @@ function install(appId, appStoreId, manifest, location, portBindings, accessRest
     });
 }
 
-function configure(appId, location, portBindings, accessRestriction, callback) {
+function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'string');
+    assert.strictEqual(typeof accessRestriction, 'object');
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof memoryLimit, 'number');
     assert.strictEqual(typeof callback, 'function');
 
     var error = validateHostname(location, config.fqdn());
@@ -340,6 +468,9 @@ function configure(appId, location, portBindings, accessRestriction, callback) {
     error = validateAccessRestriction(accessRestriction);
     if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
+    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
@@ -347,15 +478,29 @@ function configure(appId, location, portBindings, accessRestriction, callback) {
         error = validatePortBindings(portBindings, app.manifest.tcpPorts);
         if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
 
+        error = validateMemoryLimit(app.manifest, memoryLimit);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+
+        // memoryLimit might come in as 0 if not specified
+        memoryLimit = memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
+        // save cert to data/box/certs
+        if (cert && key) {
+            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        }
+
         var values = {
             location: location.toLowerCase(),
             accessRestriction: accessRestriction,
             portBindings: portBindings,
+            memoryLimit: memoryLimit,
 
             oldConfig: {
                 location: app.location,
                 accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings
+                portBindings: app.portBindings,
+                memoryLimit: app.memoryLimit
             }
         };
 
@@ -404,12 +549,30 @@ function update(appId, force, manifest, portBindings, icon, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
+        var appStoreId = app.appStoreId;
+
+        // prevent user from installing a app with different manifest id over an existing app
+        // this allows cloudron install -f --app <appid> for an app installed from the appStore
+        if (app.manifest.id !== manifest.id) {
+            if (!force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
+            // clear appStoreId so that this app does not get updates anymore. this will mark is a dev app
+            appStoreId = '';
+        }
+
+        // Ensure we update the memory limit in case the new app requires more memory as a minimum
+        var memoryLimit = manifest.memoryLimit ? (app.memoryLimit < manifest.memoryLimit ? manifest.memoryLimit : app.memoryLimit) : app.memoryLimit;
+
         var values = {
+            appStoreId: appStoreId,
             manifest: manifest,
             portBindings: portBindings,
+            memoryLimit: memoryLimit,
+
             oldConfig: {
                 manifest: app.manifest,
-                portBindings: app.portBindings
+                portBindings: app.portBindings,
+                accessRestriction: app.accessRestriction,
+                memoryLimit: app.memoryLimit
             }
         };
 
@@ -425,58 +588,48 @@ function update(appId, force, manifest, portBindings, icon, callback) {
     });
 }
 
-function getLogStream(appId, fromLine, callback) {
+function appLogFilter(app) {
-    assert.strictEqual(typeof appId, 'string');
+    var names = [ app.id ].concat(addons.getContainerNamesSync(app, app.manifest.addons));
-    assert.strictEqual(typeof fromLine, 'number'); // behaves like tail -n
-    assert.strictEqual(typeof callback, 'function');
 
-    debug('Getting logs for %s', appId);
+    return names.map(function (name) { return 'CONTAINER_NAME=' + name; });
-    appdb.get(appId, function (error, app) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
-
-        var container = docker.getContainer(app.containerId);
-        var tail = fromLine < 0 ? -fromLine : 'all';
-
-        // note: cannot access docker file directly because it needs root access
-        container.logs({ stdout: true, stderr: true, follow: true, timestamps: true, tail: tail }, function (error, logStream) {
-            if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-            var lineCount = 0;
-            var skipLinesStream = split(function mapper(line) {
-                if (++lineCount < fromLine) return undefined;
-                var timestamp = line.substr(0, line.indexOf(' ')); // sometimes this has square brackets around it
-                return JSON.stringify({ lineNumber: lineCount, timestamp: timestamp.replace(/[[\]]/g,''), log: line.substr(timestamp.length + 1) });
-            });
-            skipLinesStream.close = logStream.req.abort;
-            logStream.pipe(skipLinesStream);
-            return callback(null, skipLinesStream);
-        });
-    });
 }
 
-function getLogs(appId, callback) {
+function getLogs(appId, lines, follow, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof lines, 'number');
+    assert.strictEqual(typeof follow, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Getting logs for %s', appId);
+
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        if (app.installationState !== appdb.ISTATE_INSTALLED) return callback(new AppsError(AppsError.BAD_STATE, util.format('App is in %s state.', app.installationState)));
+        var args = [ '--output=json', '--no-pager', '--lines=' + lines ];
+        if (follow) args.push('--follow');
+        args = args.concat(appLogFilter(app));
 
-        var container = docker.getContainer(app.containerId);
+        var cp = spawn('/bin/journalctl', args);
-        // note: cannot access docker file directly because it needs root access
-        container.logs({ stdout: true, stderr: true, follow: false, timestamps: true, tail: 'all' }, function (error, logStream) {
-            if (error && error.statusCode === 404) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            return callback(null, logStream);
+        var transformStream = split(function mapper(line) {
+            var obj = safe.JSON.parse(line);
+            if (!obj) return undefined;
+
+            var source = obj.CONTAINER_NAME.slice(app.id.length + 1);
+            return JSON.stringify({
+                realtimeTimestamp: obj.__REALTIME_TIMESTAMP,
+                monotonicTimestamp: obj.__MONOTONIC_TIMESTAMP,
+                message: obj.MESSAGE,
+                source: source || 'main'
+            }) + '\n';
         });
+
+        transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process
+
+        cp.stdout.pipe(transformStream);
+
+        return callback(null, transformStream);
     });
 }
 
@@ -490,10 +643,11 @@ function restore(appId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var restoreConfig = app.lastBackupConfig;
+        // restore without a backup is the same as re-install
-        if (!restoreConfig) return callback(new AppsError(AppsError.BAD_STATE, 'No restore point'));
+        var restoreConfig = app.lastBackupConfig, values = { };
-
+        if (restoreConfig) {
-        // re-validate because this new box version may not accept old configs. if we restore location, it should be validated here as well
+            // re-validate because this new box version may not accept old configs.
+            // if we restore location, it should be validated here as well
             error = checkManifestConstraints(restoreConfig.manifest);
             if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
 
@@ -501,17 +655,20 @@ function restore(appId, callback) {
             if (error) return callback(error);
 
             // ## should probably query new location, access restriction from user
-        var values = {
+            values = {
                 manifest: restoreConfig.manifest,
                 portBindings: restoreConfig.portBindings,
+                memoryLimit: restoreConfig.memoryLimit,
 
                 oldConfig: {
                     location: app.location,
                     accessRestriction: app.accessRestriction,
                     portBindings: app.portBindings,
+                    memoryLimit: app.memoryLimit,
                     manifest: app.manifest
                 }
             };
+        }
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -530,13 +687,13 @@ function uninstall(appId, callback) {
 
     debug('Will uninstall app with id:%s', appId);
 
+    taskmanager.stopAppTask(appId, function () {
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_UNINSTALL, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        taskmanager.restartAppTask(appId); // since uninstall is allowed from any state, kill current task
+            taskmanager.startAppTask(appId, callback);
-
+        });
-        callback(null);
     });
 }
 
@@ -573,6 +730,8 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
+    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
         return new Error('Box version exceeds Apps maxBoxVersion');
     }
@@ -596,13 +755,17 @@ function exec(appId, options, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        var container = docker.getContainer(app.containerId);
+        if (app.installationState !== appdb.ISTATE_INSTALLED || app.runState !== appdb.RSTATE_RUNNING) {
+            return callback(new AppsError(AppsError.BAD_STATE, 'App not installed or running'));
+        }
+
+        var container = docker.connection.getContainer(app.containerId);
 
        var execOptions = {
             AttachStdin: true,
             AttachStdout: true,
             AttachStderr: true,
-            Tty: true,
+            Tty: options.tty,
             Cmd: cmd
         };
 
@@ -610,7 +773,7 @@ function exec(appId, options, callback) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
             var startOptions = {
                 Detach: false,
-                Tty: true,
+                Tty: options.tty,
                 stdin: true // this is a dockerode option that enabled openStdin in the modem
             };
             exec.start(startOptions, function(error, stream) {
@@ -645,27 +808,38 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     assert.strictEqual(typeof callback, 'function');
 
     function canAutoupdateApp(app, newManifest) {
-        // TODO: maybe check the description as well?
+        var tcpPorts = newManifest.tcpPorts || { };
-        if (!newManifest.tcpPorts && !app.portBindings) return true;
+        var portBindings = app.portBindings; // this is never null
-        if (!newManifest.tcpPorts || !app.portBindings) return false;
 
-        for (var env in newManifest.tcpPorts) {
+        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
-            if (!(env in app.portBindings)) return false;
+        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
+        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
+
+        for (var env in tcpPorts) {
+            if (!(env in portBindings)) return new Error(env + ' is required from user');
         }
 
-        return true;
+        // it's fine if one or more keys got removed
+        return null;
     }
 
     if (!updateInfo) return callback(null);
 
     async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
         get(appId, function (error, app) {
-            if (!canAutoupdateApp(app, updateInfo[appId].manifest)) {
+            if (error) {
+                debug('Cannot autoupdate app %s : %s', appId, error.message);
                 return iteratorDone();
            }
 
-           update(appId, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+            error = canAutoupdateApp(app, updateInfo[appId].manifest);
-                if (error) debug('Error initiating autoupdate of %s', appId);
+            if (error) {
+                debug('app %s requires manual update. %s', appId, error.message);
+                return iteratorDone();
+            }
+
+           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings, null /* icon */, function (error) {
+                if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);
 
                 iteratorDone(null);
             });
@@ -673,54 +847,93 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
     }, callback);
 }
 
+function canBackupApp(app) {
+    // only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
+    // state not good for consistent backup (i.e addons may not have been setup completely)
+    return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+            app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            app.installationState === appdb.ISTATE_PENDING_BACKUP ||  // called from apptask
+            app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
+}
+
+// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
+// s3 does not allow changing creation time, so copying the last backup is easy way out for now
+function reuseOldBackup(app, callback) {
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    backups.copyLastBackup(app, function (error, newBackupId) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
+
+        callback(null, newBackupId);
+    });
+}
+
+function createNewBackup(app, addonsToBackup, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    backups.getAppBackupUrl(app, function (error, result) {
+        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        debugApp(app, 'backupApp: backup url:%s backup config url:%s', result.url, result.configUrl);
+
+        async.series([
+            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
+            addons.backupAddons.bind(null, app, addonsToBackup),
+            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.configUrl, result.backupKey, result.sessionToken ]),
+            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
+        ], function (error) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            callback(null, result.id);
+        });
+    });
+}
+
 function backupApp(app, addonsToBackup, callback) {
     assert.strictEqual(typeof app, 'object');
     assert(!addonsToBackup || typeof addonsToBackup, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    function canBackupApp(app) {
+    var appConfig = null, backupFunction;
-        // only backup apps that are installed or pending configure. Rest of them are in some
+
-        // state not good for consistent backup (i.e addons may not have been setup completely)
+    if (!canBackupApp(app)) {
-        return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
+        if (!app.lastBackupId) {
-                app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
+            debugApp(app, 'backupApp: cannot backup app');
-                app.installationState === appdb.ISTATE_PENDING_BACKUP ||
+            return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy and never backed up previously'));
-                app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
         }
 
-    if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy'));
+        appConfig = app.lastBackupConfig;
-
+        backupFunction = reuseOldBackup.bind(null, app);
-    var appConfig = {
+    } else {
+        appConfig = {
             manifest: app.manifest,
             location: app.location,
             portBindings: app.portBindings,
-        accessRestriction: app.accessRestriction
+            accessRestriction: app.accessRestriction,
+            memoryLimit: app.memoryLimit
         };
+        backupFunction = createNewBackup.bind(null, app, addonsToBackup);
 
         if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
             return callback(safe.error);
         }
+    }
 
-    backups.getBackupUrl(app, null, function (error, result) {
+    backupFunction(function (error, backupId) {
-        if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id);
+        debugApp(app, 'backupApp: successful id:%s', backupId);
 
-        async.series([
+        setRestorePoint(app.id, backupId, appConfig, function (error) {
-            ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            addons.backupAddons.bind(null, app, addonsToBackup),
-            shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD,  app.id, result.url, result.backupKey ]),
-            ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
-        ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            debugApp(app, 'backupApp: successful id:%s', result.id);
+            return callback(null, backupId);
-
-            setRestorePoint(app.id, result.id, appConfig, function (error) {
-                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-                return callback(null, result.id);
-            });
         });
     });
 }
@@ -729,9 +942,9 @@ function backup(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    get(appId, function (error, app) {
+    appdb.exists(appId, function (error, exists) {
-        if (error && error.reason === AppsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if (!exists) return callback(new AppsError(AppsError.NOT_FOUND));
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_BACKUP, function (error) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
@@ -744,19 +957,20 @@ function backup(appId, callback) {
     });
 }
 
-function restoreApp(app, addonsToRestore, callback) {
+function restoreApp(app, addonsToRestore, backupId, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof addonsToRestore, 'object');
+    assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
     assert(app.lastBackupId);
 
-    backups.getRestoreUrl(app.lastBackupId, function (error, result) {
+    backups.getRestoreUrl(backupId, function (error, result) {
         if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
         debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
 
-        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey ], function (error) {
+        shell.sudo('restoreApp', [ RESTORE_APP_CMD,  app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
             addons.restoreAddons(app, addonsToRestore, callback);
@@ -764,3 +978,21 @@ function restoreApp(app, addonsToRestore, callback) {
     });
 }
 
+function listBackups(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.exists(appId, function (error, exists) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        if (!exists) return callback(new AppsError(AppsError.NOT_FOUND));
+
+        backups.getByAppIdPaged(page, perPage, appId, function (error, results) {
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            callback(null, results);
+        });
+    });
+}

src/apptask.js

@@ -9,7 +9,7 @@ exports = module.exports = {
     startTask: startTask,
 
     // exported for testing
-    _getFreePort: getFreePort,
+    _reserveHttpPort: reserveHttpPort,
     _configureNginx: configureNginx,
     _unconfigureNginx: unconfigureNginx,
     _createVolume: createVolume,
@@ -19,17 +19,23 @@ exports = module.exports = {
     _verifyManifest: verifyManifest,
     _registerSubdomain: registerSubdomain,
     _unregisterSubdomain: unregisterSubdomain,
-    _reloadNginx: reloadNginx,
     _waitForDnsPropagation: waitForDnsPropagation
 };
 
 require('supererror')({ splatchError: true });
 
+// remove timestamp from debug() based output
+require('debug').formatArgs = function formatArgs() {
+    arguments[0] = this.namespace + ' ' + arguments[0];
+    return arguments;
+};
+
 var addons = require('./addons.js'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
+    certificates = require('./certificates.js'),
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
     database = require('./database.js'),
@@ -41,230 +47,114 @@ var addons = require('./addons.js'),
     hat = require('hat'),
     manifestFormat = require('cloudron-manifestformat'),
     net = require('net'),
-    os = require('os'),
+    nginx = require('./nginx.js'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
+    SubdomainError = require('./subdomains.js').SubdomainError,
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
     uuid = require('node-uuid'),
-    vbox = require('./vbox.js'),
     _ = require('underscore');
 
-var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/../setup/start/nginx/appconfig.ejs', { encoding: 'utf8' }),
+var COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
-    COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd.config.ejs', { encoding: 'utf8' }),
-    RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
     RELOAD_COLLECTD_CMD = path.join(__dirname, 'scripts/reloadcollectd.sh'),
     RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh'),
     CREATEAPPDIR_CMD = path.join(__dirname, 'scripts/createappdir.sh');
 
 function initialize(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     database.initialize(callback);
 }
 
-function debugApp(app, args) {
+function debugApp(app) {
-    assert(!app || typeof app === 'object');
+    assert.strictEqual(typeof app, 'object');
 
     var prefix = app ? (app.location || '(bare)') : '(no app)';
     debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-// We expect conflicts to not happen despite closing the port (parallel app installs, app update does not reconfigure nginx etc)
+function reserveHttpPort(app, callback) {
-// https://tools.ietf.org/html/rfc6056#section-3.5 says linux uses random ephemeral port allocation
+    assert.strictEqual(typeof app, 'object');
-function getFreePort(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
     var server = net.createServer();
     server.listen(0, function () {
         var port = server.address().port;
-        server.close(function () {
+        updateApp(app, { httpPort: port }, function (error) {
-            return callback(null, port);
+            if (error) {
-        });
+                server.close();
-    });
+                return callback(error);
-}
+            }
 
-function reloadNginx(callback) {
+            server.close(callback);
-    shell.sudo('reloadNginx', [ RELOAD_NGINX_CMD ], callback);
+        });
+    });
 }
 
 function configureNginx(app, callback) {
-    getFreePort(function (error, freePort) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var vhost = config.appFqdn(app.location);
+
+    certificates.ensureCertificate(vhost, function (error, certFilePath, keyFilePath) {
         if (error) return callback(error);
 
-        var sourceDir = path.resolve(__dirname, '..');
+        nginx.configureApp(app, certFilePath, keyFilePath, callback);
-        var endpoint = app.accessRestriction ? 'oauthproxy' : 'app';
-        var nginxConf = ejs.render(NGINX_APPCONFIG_EJS, { sourceDir: sourceDir, adminOrigin: config.adminOrigin(), vhost: config.appFqdn(app.location), port: freePort, endpoint: endpoint });
-
-        var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
-        debugApp(app, 'writing config to %s', nginxConfigFilename);
-
-        if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
-            debugApp(app, 'Error creating nginx config : %s', safe.error.message);
-            return callback(safe.error);
-        }
-
-        async.series([
-            exports._reloadNginx,
-            updateApp.bind(null, app, { httpPort: freePort })
-        ], callback);
-
-        vbox.forwardFromHostToVirtualBox(app.id + '-http', freePort);
     });
 }
 
 function unconfigureNginx(app, callback) {
-    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf');
+    assert.strictEqual(typeof app, 'object');
-    if (!safe.fs.unlinkSync(nginxConfigFilename)) {
+    assert.strictEqual(typeof callback, 'function');
-        debugApp(app, 'Error removing nginx configuration : %s', safe.error.message);
-        return callback(null);
-    }
 
-    exports._reloadNginx(callback);
+    // TODO: maybe revoke the cert
-
+    nginx.unconfigureApp(app, callback);
-    vbox.unforwardFromHostToVirtualBox(app.id + '-http');
-}
-
-function downloadImage(app, callback) {
-    debugApp(app, 'downloadImage %s', app.manifest.dockerImage);
-
-    docker.pull(app.manifest.dockerImage, function (err, stream) {
-        if (err) return callback(new Error('Error connecting to docker'));
-
-        // https://github.com/dotcloud/docker/issues/1074 says each status message
-        // is emitted as a chunk
-        stream.on('data', function (chunk) {
-            var data = safe.JSON.parse(chunk) || { };
-            debugApp(app, 'downloadImage data: %j', data);
-
-            // The information here is useless because this is per layer as opposed to per image
-            if (data.status) {
-                debugApp(app, 'progress: %s', data.status); // progressDetail { current, total }
-            } else if (data.error) {
-                debugApp(app, 'error detail: %s', data.errorDetail.message);
-            }
-        });
-
-        stream.on('end', function () {
-            debugApp(app, 'download image successfully');
-
-            var image = docker.getImage(app.manifest.dockerImage);
-
-            image.inspect(function (err, data) {
-                if (err) {
-                    return callback(new Error('Error inspecting image:' + err.message));
-                }
-
-                if (!data || !data.Config) {
-                    return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
-                }
-
-                if (!data.Config.Entrypoint && !data.Config.Cmd) {
-                    return callback(new Error('Only images with entry point are allowed'));
-                }
-
-                debugApp(app, 'This image exposes ports: %j', data.Config.ExposedPorts);
-                return callback(null);
-            });
-        });
-    });
 }
 
 function createContainer(app, callback) {
-    appdb.getPortBindings(app.id, function (error, portBindings) {
+    assert.strictEqual(typeof app, 'object');
-        if (error) return callback(error);
+    assert.strictEqual(typeof callback, 'function');
+    assert(!app.containerId); // otherwise, it will trigger volumeFrom
 
-        var manifest = app.manifest;
+    debugApp(app, 'creating container');
-        var exposedPorts = {};
-        var env = [];
 
-        // docker portBindings requires ports to be exposed
+    docker.createContainer(app, function (error, container) {
-        exposedPorts[manifest.httpPort + '/tcp'] = {};
-
-        for (var e in portBindings) {
-            var hostPort = portBindings[e];
-            var containerPort = manifest.tcpPorts[e].containerPort || hostPort;
-            exposedPorts[containerPort + '/tcp'] = {};
-
-            env.push(e + '=' + hostPort);
-        }
-
-        env.push('CLOUDRON=1');
-        env.push('WEBADMIN_ORIGIN' + '=' + config.adminOrigin());
-        env.push('API_ORIGIN' + '=' + config.adminOrigin());
-
-        addons.getEnvironment(app, function (error, addonEnv) {
-            if (error) return callback(new Error('Error getting addon env: ' + error));
-
-            var containerOptions = {
-                name: app.id,
-                Hostname: config.appFqdn(app.location),
-                Tty: true,
-                Image: app.manifest.dockerImage,
-                Cmd: null,
-                Env: env.concat(addonEnv),
-                ExposedPorts: exposedPorts
-            };
-
-            debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
-
-            docker.createContainer(containerOptions, function (error, container) {
         if (error) return callback(new Error('Error creating container: ' + error));
 
         updateApp(app, { containerId: container.id }, callback);
     });
-        });
-    });
 }
 
-function deleteContainer(app, callback) {
+function deleteContainers(app, callback) {
-    if (app.containerId === null) return callback(null);
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
-    var container = docker.getContainer(app.containerId);
+    debugApp(app, 'deleting containers');
 
-    var removeOptions = {
+    docker.deleteContainers(app.id, function (error) {
-        force: true, // kill container if it's running
+        if (error) return callback(new Error('Error deleting container: ' + error));
-        v: false // removes volumes associated with the container
-    };
 
-    container.remove(removeOptions, function (error) {
+        updateApp(app, { containerId: null }, callback);
-        if (error && error.statusCode === 404) return updateApp(app, { containerId: null }, callback);
-
-        if (error) debugApp(app, 'Error removing container', error);
-        callback(error);
-    });
-}
-
-function deleteImage(app, manifest, callback) {
-    var dockerImage = manifest ? manifest.dockerImage : null;
-    if (!dockerImage) return callback(null);
-
-    docker.getImage(dockerImage).inspect(function (error, result) {
-        if (error && error.statusCode === 404) return callback(null);
-
-        if (error) return callback(error);
-
-        var removeOptions = {
-            force: true,
-            noprune: false
-        };
-
-        // delete image by id because docker pull pulls down all the tags and this is the only way to delete all tags
-        docker.getImage(result.Id).remove(removeOptions, function (error) {
-            if (error && error.statusCode === 404) return callback(null);
-            if (error && error.statusCode === 409) return callback(null); // another container using the image
-
-            if (error) debugApp(app, 'Error removing image', error);
-
-            callback(error);
-        });
     });
 }
 
 function createVolume(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     shell.sudo('createVolume', [ CREATEAPPDIR_CMD, app.id ], callback);
 }
 
 function deleteVolume(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     shell.sudo('deleteVolume', [ RMAPPDIR_CMD, app.id ], callback);
 }
 
@@ -272,22 +162,21 @@ function allocateOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!app.accessRestriction) return callback(null);
+    if (!nginx.requiresOAuthProxy(app)) return callback(null);
 
-    var appId = 'proxy-' + app.id;
+    var id = 'cid-' + uuid.v4();
-    var id = 'cid-proxy-' + uuid.v4();
     var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
-    var scope = 'profile,' + app.accessRestriction;
+    var scope = 'profile';
 
-    clientdb.add(id, appId, clientSecret, redirectURI, scope, callback);
+    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
 }
 
 function removeOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppId('proxy-' + app.id, function (error) {
+    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
         if (error && error.reason !== DatabaseError.NOT_FOUND) {
             debugApp(app, 'Error removing OAuth client id', error);
             return callback(error);
@@ -298,6 +187,9 @@ function removeOAuthProxyCredentials(app, callback) {
 }
 
 function addCollectdProfile(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     var collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId });
     fs.writeFile(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), collectdConf, function (error) {
         if (error) return callback(error);
@@ -306,83 +198,19 @@ function addCollectdProfile(app, callback) {
 }
 
 function removeCollectdProfile(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     fs.unlink(path.join(paths.COLLECTD_APPCONFIG_DIR, app.id + '.conf'), function (error) {
         if (error && error.code !== 'ENOENT') debugApp(app, 'Error removing collectd profile', error);
         shell.sudo('removeCollectdProfile', [ RELOAD_COLLECTD_CMD ], callback);
     });
 }
 
-function startContainer(app, callback) {
-    appdb.getPortBindings(app.id, function (error, portBindings) {
-        if (error) return callback(error);
-
-        var manifest = app.manifest;
-
-        var dockerPortBindings = { };
-        var isMac = os.platform() === 'darwin';
-
-        // On Mac (boot2docker), we have to export the port to external world for port forwarding from Mac to work
-        dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: isMac ? '0.0.0.0' : '127.0.0.1', HostPort: app.httpPort + '' } ];
-
-        for (var env in portBindings) {
-            var hostPort = portBindings[env];
-            var containerPort = manifest.tcpPorts[env].containerPort || hostPort;
-            dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
-            vbox.forwardFromHostToVirtualBox(app.id + '-tcp' + containerPort, hostPort);
-        }
-
-        var startOptions = {
-            Binds: addons.getBindsSync(app, app.manifest.addons),
-            PortBindings: dockerPortBindings,
-            PublishAllPorts: false,
-            Links: addons.getLinksSync(app, app.manifest.addons),
-            RestartPolicy: {
-                "Name": "always",
-                "MaximumRetryCount": 0
-            },
-            CpuShares: 512 // relative to 1024 for system processes
-        };
-
-        var container = docker.getContainer(app.containerId);
-        debugApp(app, 'Starting container %s with options: %j', container.id, JSON.stringify(startOptions));
-
-        container.start(startOptions, function (error, data) {
-            if (error && error.statusCode !== 304) return callback(new Error('Error starting container:' + error));
-
-            return callback(null);
-        });
-    });
-}
-
-function stopContainer(app, callback) {
-    var container = docker.getContainer(app.containerId);
-    debugApp(app, 'Stopping container %s', container.id);
-
-    var options = {
-        t: 10 // wait for 10 seconds before killing it
-    };
-
-    container.stop(options, function (error) {
-        if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error stopping container:' + error));
-
-        var tcpPorts = safe.query(app, 'manifest.tcpPorts', { });
-        for (var containerPort in tcpPorts) {
-            vbox.unforwardFromHostToVirtualBox(app.id + '-tcp' + containerPort);
-        }
-
-        debugApp(app, 'Waiting for container ' + container.id);
-
-        container.wait(function (error, data) {
-            if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error waiting on container:' + error));
-
-            debugApp(app, 'Container stopped with status code [%s]', data ? String(data.StatusCode) : '');
-
-            return callback(null);
-        });
-    });
-}
-
 function verifyManifest(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     debugApp(app, 'Verifying manifest');
 
     var manifest = app.manifest;
@@ -396,6 +224,9 @@ function verifyManifest(app, callback) {
 }
 
 function downloadIcon(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     debugApp(app, 'Downloading icon of %s@%s', app.appStoreId, app.manifest.version);
 
     var iconUrl = config.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon';
@@ -404,8 +235,8 @@ function downloadIcon(app, callback) {
         .get(iconUrl)
         .buffer(true)
         .end(function (error, res) {
-            if (error) return callback(new Error('Error downloading icon:' + error.message));
+            if (error && !error.response) return callback(new Error('Network error downloading icon:' + error.message));
-            if (res.status !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
+            if (res.statusCode !== 200) return callback(null); // ignore error. this can also happen for apps installed with cloudron-cli
 
             if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, app.id + '.png'), res.body)) return callback(new Error('Error saving icon:' + safe.error.message));
 
@@ -414,52 +245,64 @@ function downloadIcon(app, callback) {
 }
 
 function registerSubdomain(app, callback) {
-    debugApp(app, 'Registering subdomain location [%s]', app.location);
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    sysinfo.getIp(function (error, ip) {
+        if (error) return callback(error);
 
         // even though the bare domain is already registered in the appstore, we still
         // need to register it so that we have a dnsRecordId to wait for it to complete
-    var record = { subdomain: app.location, type: 'A', value: sysinfo.getIp() };
+        async.retry({ times: 200, interval: 5000 }, function (retryCallback) {
+            debugApp(app, 'Registering subdomain location [%s]', app.location);
 
-    superagent
+            subdomains.add(app.location, 'A', [ ip ], function (error, changeId) {
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
+                if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: [ record ] })
-        .end(function (error, res) {
-            if (error) return callback(error);
 
-            debugApp(app, 'Registered subdomain status: %s', res.status);
+                retryCallback(null, error || changeId);
+            });
+        }, function (error, result) {
+            if (error || result instanceof Error) return callback(error || result);
 
-            if (res.status === 409) return callback(null); // already registered
+            updateApp(app, { dnsRecordId: result }, callback);
-            if (res.status !== 201) return callback(new Error(util.format('Subdomain Registration failed. %s %j', res.status, res.body)));
+        });
-
-            updateApp(app, { dnsRecordId: res.body.ids[0] }, callback);
     });
 }
 
-function unregisterSubdomain(app, callback) {
+function unregisterSubdomain(app, location, callback) {
-    debugApp(app, 'Unregistering subdomain: dnsRecordId=%s', app.dnsRecordId);
+    assert.strictEqual(typeof app, 'object');
-
+    assert.strictEqual(typeof location, 'string');
-    if (!app.dnsRecordId) return callback(null);
+    assert.strictEqual(typeof callback, 'function');
 
     // do not unregister bare domain because we show a error/cloudron info page there
-    if (app.location === '') return updateApp(app, { dnsRecordId: null }, callback);
+    if (location === '') {
-
+        debugApp(app, 'Skip unregister of empty subdomain');
-    superagent
+        return callback(null);
-        .del(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId)
-        .query({ token: config.token() })
-        .end(function (error, res) {
-            if (error) {
-                debugApp(app, 'Error making request: %s', error);
-            } else if (res.status !== 204) {
-                debugApp(app, 'Error unregistering subdomain:', res.status, res.body);
     }
 
+    sysinfo.getIp(function (error, ip) {
+        if (error) return callback(error);
+
+        async.retry({ times: 30, interval: 5000 }, function (retryCallback) {
+            debugApp(app, 'Unregistering subdomain: %s', location);
+
+            subdomains.remove(location, 'A', [ ip ], function (error) {
+                if (error && (error.reason === SubdomainError.STILL_BUSY || error.reason === SubdomainError.EXTERNAL_ERROR)) return retryCallback(error); // try again
+
+                retryCallback(null, error);
+            });
+        }, function (error, result) {
+            if (error || result instanceof Error) return callback(error || result);
+
             updateApp(app, { dnsRecordId: null }, callback);
         });
+    });
 }
 
 function removeIcon(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     fs.unlink(path.join(paths.APPICONS_DIR, app.id + '.png'), function (error) {
         if (error && error.code !== 'ENOENT') debugApp(app, 'cannot remove icon : %s', error);
         callback(null);
@@ -467,6 +310,9 @@ function removeIcon(app, callback) {
 }
 
 function waitForDnsPropagation(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     if (!config.CLOUDRON) {
         debugApp(app, 'Skipping dns propagation check for development');
         return callback(null);
@@ -477,18 +323,12 @@ function waitForDnsPropagation(app, callback) {
         setTimeout(waitForDnsPropagation.bind(null, app, callback), 5000);
     }
 
-    superagent
+    subdomains.status(app.dnsRecordId, function (error, result) {
-        .get(config.apiServerOrigin() + '/api/v1/subdomains/' + app.dnsRecordId + '/status')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .end(function (error, res) {
         if (error) return retry(new Error('Failed to get dns record status : ' + error.message));
 
-            debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, res.status);
+        debugApp(app, 'waitForDnsPropagation: dnsRecordId:%s status:%s', app.dnsRecordId, result);
 
-            if (res.status !== 200) return retry(new Error(util.format('Error getting record status: %s %j', res.status, res.body)));
+        if (result !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, result)));
-
-            if (res.body.status !== 'done') return retry(new Error(util.format('app:%s not ready yet: %s', app.id, res.body.status)));
 
         callback(null);
     });
@@ -496,7 +336,11 @@ function waitForDnsPropagation(app, callback) {
 
 // updates the app object and the database
 function updateApp(app, values, callback) {
-    debugApp(app, 'installationState: %s progress: %s', app.installationState, app.installationProgress);
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof values, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'updating app with values: %j', values);
 
     appdb.update(app.id, values, function (error) {
         if (error) return callback(error);
@@ -520,23 +364,25 @@ function updateApp(app, values, callback) {
 //   - setup the container (requires image, volumes, addons)
 //   - setup collectd (requires container id)
 function install(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     async.series([
         verifyManifest.bind(null, app),
 
         // teardown for re-installs
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        unconfigureNginx.bind(null, app),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
         deleteVolume.bind(null, app),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
         removeOAuthProxyCredentials.bind(null, app),
-        removeIcon.bind(null, app),
+        // removeIcon.bind(null, app), // do not remove icon for non-appstore installs
-        unconfigureNginx.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '15, Configure nginx' }),
+        reserveHttpPort.bind(null, app),
-        configureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '20, Downloading icon' }),
         downloadIcon.bind(null, app),
@@ -548,7 +394,7 @@ function install(app, callback) {
         registerSubdomain.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '40, Downloading image' }),
-        downloadImage.bind(null, app),
+        docker.downloadImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '50, Creating volume' }),
         createVolume.bind(null, app),
@@ -567,6 +413,9 @@ function install(app, callback) {
         updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
+        configureNginx.bind(null, app),
+
         // done!
         function (callback) {
             debugApp(app, 'installed');
@@ -582,6 +431,9 @@ function install(app, callback) {
 }
 
 function backup(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
         apps.backupApp.bind(null, app, app.manifest.addons),
@@ -602,6 +454,9 @@ function backup(app, callback) {
 
 // restore is also called for upgrades and infra updates. note that in those cases it is possible there is no backup
 function restore(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     // we don't have a backup, same as re-install. this allows us to install from install failures (update failures always
     // have a backupId)
     if (!app.lastBackupId) {
@@ -609,21 +464,26 @@ function restore(app, callback) {
         return install(app, callback);
     }
 
+    var backupId = app.lastBackupId;
+
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        unconfigureNginx.bind(null, app),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
          // oldConfig can be null during upgrades
         addons.teardownAddons.bind(null, app, app.oldConfig ? app.oldConfig.manifest.addons : null),
         deleteVolume.bind(null, app),
-        deleteImage.bind(null, app, app.manifest),
+        function deleteImageIfChanged(done) {
+             if (!app.oldConfig || (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage)) return done();
+
+             docker.deleteImage(app.oldConfig.manifest, done);
+        },
         removeOAuthProxyCredentials.bind(null, app),
         removeIcon.bind(null, app),
-        unconfigureNginx.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '30, Configuring Nginx' }),
+        reserveHttpPort.bind(null, app),
-        configureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '40, Downloading icon' }),
         downloadIcon.bind(null, app),
@@ -635,13 +495,13 @@ function restore(app, callback) {
         registerSubdomain.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '60, Downloading image' }),
-        downloadImage.bind(null, app),
+        docker.downloadImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '65, Creating volume' }),
         createVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
-        apps.restoreApp.bind(null, app, app.manifest.addons),
+        apps.restoreApp.bind(null, app, app.manifest.addons, backupId),
 
         updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
         createContainer.bind(null, app),
@@ -654,6 +514,9 @@ function restore(app, callback) {
         updateApp.bind(null, app, { installationProgress: '90, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
+        updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
+        configureNginx.bind(null, app),
+
         // done!
         function (callback) {
             debugApp(app, 'restored');
@@ -671,34 +534,29 @@ function restore(app, callback) {
 
 // note that configure is called after an infra update as well
 function configure(app, callback) {
-    var locationChanged = app.oldConfig ? app.oldConfig.location !== app.location : true;
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        unconfigureNginx.bind(null, app),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
         function (next) {
-            if (!locationChanged) return next();
+            // oldConfig can be null during an infra update
-            unregisterSubdomain(app, next);
+            if (!app.oldConfig || app.oldConfig.location === app.location) return next();
+            unregisterSubdomain(app, app.oldConfig.location, next);
         },
         removeOAuthProxyCredentials.bind(null, app),
-        unconfigureNginx.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '25, Configuring Nginx' }),
+        reserveHttpPort.bind(null, app),
-        configureNginx.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '30, Create OAuth proxy credentials' }),
         allocateOAuthProxyCredentials.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
         updateApp.bind(null, app, { installationProgress: '35, Registering subdomain' }),
-                registerSubdomain.bind(null, app)
+        registerSubdomain.bind(null, app),
-            ], next);
-        },
 
         // re-setup addons since they rely on the app's fqdn (e.g oauth)
         updateApp.bind(null, app, { installationProgress: '50, Setting up addons' }),
@@ -712,14 +570,11 @@ function configure(app, callback) {
 
         runApp.bind(null, app),
 
-        function (next) {
-            if (!locationChanged) return next();
-
-            async.series([
         updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
-                exports._waitForDnsPropagation.bind(null, app)
+        exports._waitForDnsPropagation.bind(null, app),
-            ], next);
+
-        },
+        updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
+        configureNginx.bind(null, app),
 
         // done!
         function (callback) {
@@ -737,40 +592,50 @@ function configure(app, callback) {
 
 // nginx configuration is skipped because app.httpPort is expected to be available
 function update(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     debugApp(app, 'Updating to %s', safe.query(app, 'manifest.version'));
 
     // app does not want these addons anymore
+    // FIXME: this does not handle option changes (like multipleDatabases)
     var unusedAddons = _.omit(app.oldConfig.manifest.addons, Object.keys(app.manifest.addons));
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '0, Verify manifest' }),
         verifyManifest.bind(null, app),
 
+        // download new image before app is stopped. this is so we can reduce downtime
+        // and also not remove the 'common' layers when the old image is deleted
+        updateApp.bind(null, app, { installationProgress: '15, Downloading image' }),
+        docker.downloadImage.bind(null, app.manifest),
+
         // note: we cleanup first and then backup. this is done so that the app is not running should backup fail
         // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings
-        updateApp.bind(null, app, { installationProgress: '10, Cleaning up old install' }),
+        updateApp.bind(null, app, { installationProgress: '25, Cleaning up old install' }),
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
         addons.teardownAddons.bind(null, app, unusedAddons),
-        deleteImage.bind(null, app, app.manifest), // delete image even if did not change (see df158b111f)
+        function deleteImageIfChanged(done) {
+             if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
+
+             docker.deleteImage(app.oldConfig.manifest, done);
+        },
         // removeIcon.bind(null, app), // do not remove icon, otherwise the UI breaks for a short time...
 
         function (next) {
             if (app.installationState === appdb.ISTATE_PENDING_FORCE_UPDATE) return next(null);
 
             async.series([
-                updateApp.bind(null, app, { installationProgress: '20, Backup app' }),
+                updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
                 apps.backupApp.bind(null, app, app.oldConfig.manifest.addons)
             ], next);
         },
 
-        updateApp.bind(null, app, { installationProgress: '35, Downloading icon' }),
+        updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '45, Downloading image' }),
-        downloadImage.bind(null, app),
-
         updateApp.bind(null, app, { installationProgress: '70, Updating addons' }),
         addons.setupAddons.bind(null, app, app.manifest.addons),
 
@@ -797,6 +662,9 @@ function update(app, callback) {
 }
 
 function uninstall(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     debugApp(app, 'uninstalling');
 
     async.series([
@@ -807,7 +675,7 @@ function uninstall(app, callback) {
         stopApp.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '20, Deleting container' }),
-        deleteContainer.bind(null, app),
+        deleteContainers.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '30, Teardown addons' }),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
@@ -816,10 +684,10 @@ function uninstall(app, callback) {
         deleteVolume.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '50, Deleting image' }),
-        deleteImage.bind(null, app, app.manifest),
+        docker.deleteImage.bind(null, app.manifest),
 
         updateApp.bind(null, app, { installationProgress: '60, Unregistering subdomain' }),
-        unregisterSubdomain.bind(null, app),
+        unregisterSubdomain.bind(null, app, app.location),
 
         updateApp.bind(null, app, { installationProgress: '70, Remove OAuth credentials' }),
         removeOAuthProxyCredentials.bind(null, app),
@@ -832,29 +700,41 @@ function uninstall(app, callback) {
 
         updateApp.bind(null, app, { installationProgress: '95, Remove app from database' }),
         appdb.del.bind(null, app.id)
-    ], callback);
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error uninstalling app: %s', error);
+            return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
+        }
+        callback(null);
+    });
 }
 
 function runApp(app, callback) {
-    startContainer(app, function (error) {
+    assert.strictEqual(typeof app, 'object');
-        if (error) {
+    assert.strictEqual(typeof callback, 'function');
-            debugApp(app, 'Error starting container : %s', error);
+
-            return updateApp(app, { runState: appdb.RSTATE_ERROR }, callback);
+    docker.startContainer(app.containerId, function (error) {
-        }
+        if (error) return callback(error);
 
         updateApp(app, { runState: appdb.RSTATE_RUNNING }, callback);
     });
 }
 
 function stopApp(app, callback) {
-    stopContainer(app, function (error) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    docker.stopContainers(app.id, function (error) {
         if (error) return callback(error);
 
-        updateApp(app, { runState: appdb.RSTATE_STOPPED }, callback);
+        updateApp(app, { runState: appdb.RSTATE_STOPPED, health: null }, callback);
     });
 }
 
 function handleRunCommand(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
     if (app.runState === appdb.RSTATE_PENDING_STOP) {
         return stopApp(app, callback);
     }
@@ -870,6 +750,9 @@ function handleRunCommand(app, callback) {
 }
 
 function startTask(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
     // determine what to do
     appdb.get(appId, function (error, app) {
         if (error) return callback(error);
@@ -886,7 +769,7 @@ function startTask(appId, callback) {
         case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
         case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
         case appdb.ISTATE_ERROR:
-            debugApp(app, 'Apptask launched with error states.');
+            debugApp(app, 'Internal error. apptask launched with error status.');
             return callback(null);
         default:
             debugApp(app, 'apptask launched with invalid command');
@@ -904,7 +787,7 @@ if (require.main === module) {
         if (error) throw error;
 
         startTask(process.argv[2], function (error) {
-            if (error) console.error(error);
+            if (error) debug('Apptask completed with error', error);
 
             debug('Apptask completed for %s', process.argv[2]);
             // https://nodejs.org/api/process.html are exit codes used by node. apps.js uses the value below
@@ -913,4 +796,3 @@ if (require.main === module) {
         });
     });
 }
-

src/auth.js

@@ -16,6 +16,7 @@ var assert = require('assert'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
     crypto = require('crypto'),
+    groups = require('./groups'),
     passport = require('passport'),
     tokendb = require('./tokendb'),
     user = require('./user'),
@@ -123,9 +124,16 @@ function initialize(callback) {
                 // amend the tokenType of the token owner
                 user.tokenType = tokenType;
 
+                // amend the admin flag
+                groups.isMember(groups.ADMIN_GROUP_ID, user.id, function (error, isAdmin) {
+                    if (error) return callback(error);
+
+                    user.admin = isAdmin;
+
                     callback(null, user, info);
                 });
             });
+        });
     }));
 
     callback(null);

src/backupdb.js

@@ -0,0 +1,116 @@
+/* jslint node:true */
+
+'use strict';
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var BACKUPS_FIELDS = [ 'filename', 'creationTime', 'version', 'type', 'dependsOn', 'state' ];
+
+exports = module.exports = {
+    add: add,
+    getPaged: getPaged,
+    get: get,
+    del: del,
+    getByAppIdPaged: getByAppIdPaged,
+
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+};
+
+function postProcess(result) {
+    assert.strictEqual(typeof result, 'object');
+
+    result.dependsOn = result.dependsOn ? result.dependsOn.split(',') : [ ];
+}
+
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL, (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function getByAppIdPaged(page, perPage, appId, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND filename LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, 'appbackup\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(function (result) { postProcess(result); });
+
+        callback(null, results);
+    });
+}
+
+function get(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE filename = ? AND type = ? AND state = ? ORDER BY creationTime DESC',
+        [ filename, exports.BACKUP_TYPE_BOX, exports.BACKUP_STATE_NORMAL ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(result[0]);
+
+        callback(null, result[0]);
+    });
+}
+
+function add(backup, callback) {
+    assert(backup && typeof backup === 'object');
+    assert.strictEqual(typeof backup.filename, 'string');
+    assert.strictEqual(typeof backup.version, 'string');
+    assert(backup.type === exports.BACKUP_TYPE_APP || backup.type === exports.BACKUP_TYPE_BOX);
+    assert(util.isArray(backup.dependsOn));
+    assert.strictEqual(typeof callback, 'function');
+
+    var creationTime = backup.creationTime || new Date(); // allow tests to set the time
+
+    database.query('INSERT INTO backups (filename, version, type, creationTime, state, dependsOn) VALUES (?, ?, ?, ?, ?, ?)',
+        [ backup.filename, backup.version, backup.type, creationTime, exports.BACKUP_STATE_NORMAL, backup.dependsOn.join(',') ],
+        function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE backups', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(filename, callback) {
+    assert.strictEqual(typeof filename, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM backups WHERE filename=?', [ filename ], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}

src/backups.js

@@ -3,16 +3,23 @@
 exports = module.exports = {
     BackupsError: BackupsError,
 
-    getAllPaged: getAllPaged,
+    getPaged: getPaged,
+    getByAppIdPaged: getByAppIdPaged,
 
     getBackupUrl: getBackupUrl,
-    getRestoreUrl: getRestoreUrl
+    getAppBackupUrl: getAppBackupUrl,
+    getRestoreUrl: getRestoreUrl,
+
+    copyLastBackup: copyLastBackup
 };
 
 var assert = require('assert'),
+    backupdb = require('./backupdb.js'),
+    caas = require('./storage/caas.js'),
     config = require('./config.js'),
     debug = require('debug')('box:backups'),
-    superagent = require('superagent'),
+    s3 = require('./storage/s3.js'),
+    settings = require('./settings.js'),
     util = require('util');
 
 function BackupsError(reason, errorOrMessage) {
@@ -36,60 +43,158 @@ function BackupsError(reason, errorOrMessage) {
 util.inherits(BackupsError, Error);
 BackupsError.EXTERNAL_ERROR = 'external error';
 BackupsError.INTERNAL_ERROR = 'internal error';
+BackupsError.MISSING_CREDENTIALS = 'missing credentials';
 
-function getAllPaged(page, perPage, callback) {
+// choose which storage backend we use for test purpose we use s3
-    assert.strictEqual(typeof page, 'number');
+function api(provider) {
-    assert.strictEqual(typeof perPage, 'number');
+    switch (provider) {
+        case 'caas': return caas;
+        case 's3': return s3;
+        default: return null;
+    }
+}
+
+function getPaged(page, perPage, callback) {
+    assert(typeof page === 'number' && page > 0);
+    assert(typeof perPage === 'number' && perPage > 0);
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backups';
+    backupdb.getPaged(page, perPage, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    superagent.get(url).query({ token: config.token() }).end(function (error, result) {
+        callback(null, results);
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 200) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !util.isArray(result.body.backups)) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
-
-        // [ { creationTime, boxVersion, restoreKey, dependsOn: [ ] } ] sorted by time (latest first)
-        return callback(null, result.body.backups);
     });
 }
 
-function getBackupUrl(app, appBackupIds, callback) {
+function getByAppIdPaged(page, perPage, appId, callback) {
-    assert(!app || typeof app === 'object');
+    assert(typeof page === 'number' && page > 0);
-    assert(!appBackupIds || util.isArray(appBackupIds));
+    assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/backupurl';
+    backupdb.getByAppIdPaged(page, perPage, appId, function (error, results) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    var data = {
+        callback(null, results);
-        boxVersion: config.version(),
+    });
-        appId: app ? app.id : null,
+}
-        appVersion: app ? app.manifest.version : null,
+
-        appBackupIds: appBackupIds
+function getBackupUrl(appBackupIds, callback) {
+    assert(util.isArray(appBackupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('backup_%s-v%s', now.toISOString(), config.version());
+    var filename = filebase + '.tar.gz';
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getSignedUploadUrl(backupConfig, filename, function (error, result) {
+            if (error) return callback(error);
+
+            var obj = {
+                id: filename,
+                url: result.url,
+                sessionToken: result.sessionToken,
+                backupKey: backupConfig.key
             };
 
-    superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
+            debug('getBackupUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
 
-        return callback(null, result.body);
+            backupdb.add({ filename: filename, creationTime: now, version: config.version(), type: backupdb.BACKUP_TYPE_BOX, dependsOn: appBackupIds }, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                callback(null, obj);
+            });
+        });
     });
 }
 
+function getAppBackupUrl(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var now = new Date();
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
+    var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
+ 
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getSignedUploadUrl(backupConfig, configFilename, function (error, configResult) {
+            if (error) return callback(error);
+
+            api(backupConfig.provider).getSignedUploadUrl(backupConfig, dataFilename, function (error, dataResult) {
+                if (error) return callback(error);
+
+                var obj = {
+                    id: dataFilename,
+                    url: dataResult.url,
+                    configUrl: configResult.url,
+                    sessionToken: dataResult.sessionToken, // this token can be used for both config and data upload
+                    backupKey: backupConfig.key // only data is encrypted
+                };
+
+                debug('getAppBackupUrl: %j', obj);
+
+                backupdb.add({ filename: dataFilename, creationTime: now, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+                    if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+                    callback(null, obj);
+                });
+            });
+        });
+    });
+}
+
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
-    superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
+        api(backupConfig.provider).getSignedDownloadUrl(backupConfig, backupId, function (error, result) {
-        if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+            if (error) return callback(error);
-        if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
-        if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
 
-        return callback(null, result.body);
+            var obj = {
+                id: backupId,
+                url: result.url,
+                sessionToken: result.sessionToken,
+                backupKey: backupConfig.key
+            };
+
+            debug('getRestoreUrl: id:%s url:%s sessionToken:%s backupKey:%s', obj.id, obj.url, obj.sessionToken, obj.backupKey);
+
+            callback(null, obj);
+        });
     });
 }
 
+function copyLastBackup(app, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, (new Date()).toISOString(), app.manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, (new Date()).toISOString(), app.manifest.version);
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).copyObject(backupConfig, app.lastBackupId, toFilenameArchive, function (error) {
+            if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+            // TODO change that logic by adjusting app.lastBackupId to not contain the file type
+            var configFileId = app.lastBackupId.slice(0, -'.tar.gz'.length) + '.json';
+
+            api(backupConfig.provider).copyObject(backupConfig, configFileId, toFilenameConfig, function (error) {
+                if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
+
+                return callback(null, toFilenameArchive);
+            });
+        });
+    });
+}

src/cert/acme.js

@@ -0,0 +1,467 @@
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:cert/acme'),
+    fs = require('fs'),
+    parseLinks = require('parse-links'),
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
+    superagent = require('superagent'),
+    ursa = require('ursa'),
+    util = require('util'),
+    _ = require('underscore');
+
+var CA_PROD = 'https://acme-v01.api.letsencrypt.org',
+    CA_STAGING = 'https://acme-staging.api.letsencrypt.org',
+    LE_AGREEMENT = 'https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf';
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+function AcmeError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(AcmeError, Error);
+AcmeError.INTERNAL_ERROR = 'Internal Error';
+AcmeError.EXTERNAL_ERROR = 'External Error';
+AcmeError.ALREADY_EXISTS = 'Already Exists';
+AcmeError.NOT_COMPLETED = 'Not Completed';
+AcmeError.FORBIDDEN = 'Forbidden';
+
+// http://jose.readthedocs.org/en/latest/
+// https://www.ietf.org/proceedings/92/slides/slides-92-acme-1.pdf
+// https://community.letsencrypt.org/t/list-of-client-implementations/2103
+
+function Acme(options) {
+    assert.strictEqual(typeof options, 'object');
+
+    this.caOrigin = options.prod ? CA_PROD : CA_STAGING;
+    this.accountKeyPem = null; // Buffer
+    this.email = options.email;
+}
+
+Acme.prototype.getNonce = function (callback) {
+    superagent.get(this.caOrigin + '/directory', function (error, response) {
+        if (error) return callback(error);
+        if (response.statusCode !== 200) return callback(new Error('Invalid response code when fetching nonce : ' + response.statusCode));
+
+        return callback(null, response.headers['Replay-Nonce'.toLowerCase()]);
+    });
+};
+
+// urlsafe base64 encoding (jose)
+function urlBase64Encode(string) {
+    return string.replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '');
+}
+
+function b64(str) {
+    var buf = util.isBuffer(str) ? str : new Buffer(str);
+   return urlBase64Encode(buf.toString('base64'));
+}
+
+Acme.prototype.sendSignedRequest = function (url, payload, callback) {
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof payload, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    assert(util.isBuffer(this.accountKeyPem));
+    var privateKey = ursa.createPrivateKey(this.accountKeyPem);
+
+    var header = {
+        alg: 'RS256',
+        jwk: {
+            e: b64(privateKey.getExponent()),
+            kty: 'RSA',
+            n: b64(privateKey.getModulus())
+        }
+    };
+ 
+    var payload64 = b64(payload);
+
+    this.getNonce(function (error, nonce) {
+        if (error) return callback(error);
+
+        debug('sendSignedRequest: using nonce %s for url %s', nonce, url);
+
+        var protected64 = b64(JSON.stringify(_.extend({ }, header, { nonce: nonce })));
+
+        var signer = ursa.createSigner('sha256');
+        signer.update(protected64 + '.' + payload64, 'utf8');
+        var signature64 = urlBase64Encode(signer.sign(privateKey, 'base64'));
+
+        var data = {
+            header: header,
+            protected: protected64,
+            payload: payload64,
+            signature: signature64
+        };
+
+        superagent.post(url).set('Content-Type', 'application/x-www-form-urlencoded').send(JSON.stringify(data)).end(function (error, res) {
+            if (error && !error.response) return callback(error); // network errors
+
+            callback(null, res);
+        });
+    });
+};
+
+Acme.prototype.updateContact = function (registrationUri, callback) {
+    assert.strictEqual(typeof registrationUri, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('updateContact: %s %s', registrationUri, this.email);
+
+    // https://github.com/ietf-wg-acme/acme/issues/30
+    var payload = {
+        resource: 'reg',
+        contact: [ 'mailto:' + this.email ],
+        agreement: LE_AGREEMENT
+    };
+
+    var that = this;
+    this.sendSignedRequest(registrationUri, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
+        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to update contact. Expecting 202, got %s %s', result.statusCode, result.text)));
+
+        debug('updateContact: contact of user updated to %s', that.email);
+
+        callback();
+    });
+};
+
+Acme.prototype.registerUser = function (callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-reg',
+        contact: [ 'mailto:' + this.email ],
+        agreement: LE_AGREEMENT
+    };
+
+    debug('registerUser: %s', this.email);
+
+    var that = this;
+    this.sendSignedRequest(this.caOrigin + '/acme/new-reg', JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering user: ' + error.message));
+        if (result.statusCode === 409) return that.updateContact(result.headers.location, callback); // already exists
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerUser: registered user %s', that.email);
+
+        callback(null);
+    });
+};
+
+Acme.prototype.registerDomain = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var payload = {
+        resource: 'new-authz',
+        identifier: {
+            type: 'dns',
+            value: domain
+        }
+    };
+
+    debug('registerDomain: %s', domain);
+
+    this.sendSignedRequest(this.caOrigin + '/acme/new-authz', JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when registering domain: ' + error.message));
+        if (result.statusCode === 403) return callback(new AcmeError(AcmeError.FORBIDDEN, result.body.detail));
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to register user. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        debug('registerDomain: registered %s', domain);
+
+        callback(null, result.body);
+    });
+};
+
+Acme.prototype.prepareHttpChallenge = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('prepareHttpChallenge: preparing for challenge %j', challenge);
+
+    var token = challenge.token;
+
+    assert(util.isBuffer(this.accountKeyPem));
+    var privateKey = ursa.createPrivateKey(this.accountKeyPem);
+
+    var jwk = {
+        e: b64(privateKey.getExponent()),
+        kty: 'RSA',
+        n: b64(privateKey.getModulus())
+    };
+
+    var shasum = crypto.createHash('sha256');
+    shasum.update(JSON.stringify(jwk));
+    var thumbprint = urlBase64Encode(shasum.digest('base64'));
+    var keyAuthorization = token + '.' + thumbprint;
+
+    debug('prepareHttpChallenge: writing %s to %s', keyAuthorization, path.join(paths.ACME_CHALLENGES_DIR, token));
+
+    fs.writeFile(path.join(paths.ACME_CHALLENGES_DIR, token), token + '.' + thumbprint, function (error) {
+        if (error) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, error));
+
+        callback();
+    });
+};
+
+Acme.prototype.notifyChallengeReady = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('notifyChallengeReady: %s was met', challenge.uri);
+
+    var keyAuthorization = fs.readFileSync(path.join(paths.ACME_CHALLENGES_DIR, challenge.token), 'utf8');
+
+    var payload = {
+        resource: 'challenge',
+        keyAuthorization: keyAuthorization
+    };
+
+    this.sendSignedRequest(challenge.uri, JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when notifying challenge: ' + error.message));
+        if (result.statusCode !== 202) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to notify challenge. Expecting 202, got %s %s', result.statusCode, result.text)));
+
+        callback();
+    });
+};
+
+Acme.prototype.waitForChallenge = function (challenge, callback) {
+    assert.strictEqual(typeof challenge, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('waitingForChallenge: %j', challenge);
+
+    async.retry({ times: 10, interval: 5000 }, function (retryCallback) {
+        debug('waitingForChallenge: getting status');
+
+        superagent.get(challenge.uri).end(function (error, result) {
+            if (error && !error.response) {
+                debug('waitForChallenge: network error getting uri %s', challenge.uri);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, error.message)); // network error
+            }
+            if (result.statusCode !== 202) {
+                debug('waitForChallenge: invalid response code getting uri %s', result.statusCode);
+                return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Bad response code:' + result.statusCode));
+            }
+
+            debug('waitForChallenge: status is "%s"', result.body.status);
+
+            if (result.body.status === 'pending') return retryCallback(new AcmeError(AcmeError.NOT_COMPLETED));
+            else if (result.body.status === 'valid') return retryCallback();
+            else return retryCallback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Unexpected status: ' + result.body.status));
+        });
+    }, function retryFinished(error) {
+        // async.retry will pass 'undefined' as second arg making it unusable with async.waterfall()
+        callback(error);
+    });
+};
+
+// https://community.letsencrypt.org/t/public-beta-rate-limits/4772 for rate limits
+Acme.prototype.signCertificate = function (domain, csrDer, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert(util.isBuffer(csrDer));
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+
+    var payload = {
+        resource: 'new-cert',
+        csr: b64(csrDer)
+    };
+
+    debug('signCertificate: sending new-cert request');
+
+    this.sendSignedRequest(this.caOrigin + '/acme/new-cert', JSON.stringify(payload), function (error, result) {
+        if (error) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when signing certificate: ' + error.message));
+        // 429 means we reached the cert limit for this domain
+        if (result.statusCode !== 201) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to sign certificate. Expecting 201, got %s %s', result.statusCode, result.text)));
+
+        var certUrl = result.headers.location;
+
+        if (!certUrl) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Missing location in downloadCertificate'));
+
+        safe.fs.writeFileSync(path.join(outdir, domain + '.url'), certUrl, 'utf8'); // maybe use for renewal
+
+        return callback(null, result.headers.location);
+    });
+};
+
+Acme.prototype.createKeyAndCsr = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+    var csrFile = path.join(outdir, domain + '.csr');
+    var privateKeyFile = path.join(outdir, domain + '.key');
+    var execSync = safe.child_process.execSync;
+
+    if (safe.fs.existsSync(privateKeyFile)) {
+        // in some old releases, csr file was corrupt. so always regenerate it
+        debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
+    } else {
+        var key = execSync('openssl genrsa 4096');
+        if (!key) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+        if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        debug('createKeyAndCsr: key file saved at %s', privateKeyFile);
+    }
+
+    var csrDer = execSync(util.format('openssl req -new -key %s -outform DER -subj /CN=%s', privateKeyFile, domain));
+    if (!csrDer) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+    if (!safe.fs.writeFileSync(csrFile, csrDer)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error)); // bookkeeping
+
+    debug('createKeyAndCsr: csr file (DER) saved at %s', csrFile);
+
+    callback(null, csrDer);
+};
+
+// TODO: download the chain in a loop following 'up' header
+Acme.prototype.downloadChain = function (linkHeader, callback) {
+    if (!linkHeader) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Empty link header when downloading certificate chain');
+
+    var linkInfo = parseLinks(linkHeader);
+    if (!linkInfo || !linkInfo.up) return new AcmeError(AcmeError.EXTERNAL_ERROR, 'Failed to parse link header when downloading certificate chain'); 
+
+    debug('downloadChain: downloading from %s', this.caOrigin + linkInfo.up);
+
+    superagent.get(this.caOrigin + linkInfo.up).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var chainDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        var chainPem = execSync('openssl x509 -inform DER -outform PEM', { input: chainDer }); // this is really just base64 encoding with header
+        if (!chainPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        callback(null, chainPem);
+    });
+};
+
+Acme.prototype.downloadCertificate = function (domain, certUrl, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof certUrl, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var outdir = paths.APP_CERTS_DIR;
+    var that = this;
+
+    superagent.get(certUrl).buffer().parse(function (res, done) {
+        var data = [ ];
+        res.on('data', function(chunk) { data.push(chunk); });
+        res.on('end', function () { res.text = Buffer.concat(data); done(); });
+    }).end(function (error, result) {
+        if (error && !error.response) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'Network error when downloading certificate'));
+        if (result.statusCode === 202) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, 'Retry not implemented yet'));
+        if (result.statusCode !== 200) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, util.format('Failed to get cert. Expecting 200, got %s %s', result.statusCode, result.text)));
+
+        var certificateDer = result.text;
+        var execSync = safe.child_process.execSync;
+
+        safe.fs.writeFileSync(path.join(outdir, domain + '.der'), certificateDer);
+        debug('downloadCertificate: cert der file for %s saved', domain);
+
+        var certificatePem = execSync('openssl x509 -inform DER -outform PEM', { input: certificateDer }); // this is really just base64 encoding with header
+        if (!certificatePem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        that.downloadChain(result.header['link'], function (error, chainPem) {
+            if (error) return callback(error);
+
+            var certificateFile = path.join(outdir, domain + '.cert');
+            var fullChainPem = Buffer.concat([certificatePem, chainPem]);
+            if (!safe.fs.writeFileSync(certificateFile, fullChainPem)) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+            debug('downloadCertificate: cert file for %s saved at %s', domain, certificateFile);
+
+            callback();
+        });
+    });
+};
+
+Acme.prototype.acmeFlow = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!fs.existsSync(paths.ACME_ACCOUNT_KEY_FILE)) {
+        debug('getCertificate: generating acme account key on first run');
+        this.accountKeyPem = safe.child_process.execSync('openssl genrsa 4096');
+        if (!this.accountKeyPem) return callback(new AcmeError(AcmeError.INTERNAL_ERROR, safe.error));
+
+        safe.fs.writeFileSync(paths.ACME_ACCOUNT_KEY_FILE, this.accountKeyPem);
+    } else {
+        debug('getCertificate: using existing acme account key');
+        this.accountKeyPem = fs.readFileSync(paths.ACME_ACCOUNT_KEY_FILE);
+    }
+
+    var that = this;
+    this.registerUser(function (error) {
+        if (error) return callback(error);
+
+        that.registerDomain(domain, function (error, result) {
+            if (error) return callback(error);
+
+            debug('acmeFlow: challenges: %j', result);
+
+            var httpChallenges = result.challenges.filter(function(x) { return x.type === 'http-01'; });
+            if (httpChallenges.length === 0) return callback(new AcmeError(AcmeError.EXTERNAL_ERROR, 'no http challenges'));
+            var challenge = httpChallenges[0];
+
+            async.waterfall([
+                that.prepareHttpChallenge.bind(that, challenge),
+                that.notifyChallengeReady.bind(that, challenge),
+                that.waitForChallenge.bind(that, challenge),
+                that.createKeyAndCsr.bind(that, domain),
+                that.signCertificate.bind(that, domain),
+                that.downloadCertificate.bind(that, domain)
+            ], callback);
+        });
+    });
+};
+
+Acme.prototype.getCertificate = function (domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: start acme flow for %s from %s', domain, this.caOrigin);
+    this.acmeFlow(domain, function (error) {
+        if (error) return callback(error);
+
+        var outdir = paths.APP_CERTS_DIR;
+        callback(null, path.join(outdir, domain + '.cert'), path.join(outdir, domain + '.key'));
+    });
+};
+
+function getCertificate(domain, options, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var acme = new Acme(options || { });
+    acme.getCertificate(domain, callback);
+}

src/cert/caas.js

@@ -0,0 +1,18 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+var assert = require('assert'),
+	debug = require('debug')('box:cert/caas.js');
+
+function getCertificate(domain, options, callback) {
+	assert.strictEqual(typeof domain, 'string');
+	assert.strictEqual(typeof options, 'object');
+	assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: using fallback certificate', domain);
+
+    return callback(null, 'cert/host.cert', 'cert/host.key');
+}

src/certificates.js

@@ -0,0 +1,304 @@
+'use strict';
+
+var acme = require('./cert/acme.js'),
+    apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    caas = require('./cert/caas.js'),
+    cloudron = require('./cloudron.js'),
+    config = require('./config.js'),
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/certificates'),
+    fs = require('fs'),
+    mailer = require('./mailer.js'),
+    nginx = require('./nginx.js'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    settings = require('./settings.js'),
+    sysinfo = require('./sysinfo.js'),
+    tld = require('tldjs'),
+    user = require('./user.js'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js'),
+    x509 = require('x509');
+
+exports = module.exports = {
+    installAdminCertificate: installAdminCertificate,
+    autoRenew: autoRenew,
+    setFallbackCertificate: setFallbackCertificate,
+    setAdminCertificate: setAdminCertificate,
+    CertificatesError: CertificatesError,
+    validateCertificate: validateCertificate,
+    ensureCertificate: ensureCertificate
+};
+
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
+function CertificatesError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(CertificatesError, Error);
+CertificatesError.INTERNAL_ERROR = 'Internal Error';
+CertificatesError.INVALID_CERT = 'Invalid certificate';
+
+function getApi(callback) {
+    settings.getTlsConfig(function (error, tlsConfig) {
+        if (error) return callback(error);
+
+        var api = tlsConfig.provider === 'caas' ? caas : acme;
+
+        var options = { };
+        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+
+        // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
+        // we cannot use admin@fqdn because the user might not have set it up.
+        // we simply update the account with the latest email we have each time when getting letsencrypt certs
+        // https://github.com/ietf-wg-acme/acme/issues/30
+        user.getOwner(function (error, owner) {
+            options.email = error ? 'admin@cloudron.io' : owner.email; // can error if not activated yet
+
+            callback(null, api, options);
+        });
+    });
+}
+
+function installAdminCertificate(callback) {
+    if (cloudron.isConfiguredSync()) return callback();
+
+    settings.getTlsConfig(function (error, tlsConfig) {
+        if (error) return callback(error);
+
+        if (tlsConfig.provider === 'caas') return callback();
+
+        sysinfo.getIp(function (error, ip) {
+            if (error) return callback(error);
+
+            var zoneName = tld.getDomain(config.fqdn());
+            waitForDns(config.adminFqdn(), ip, zoneName, function (error) {
+                if (error) return callback(error); // this cannot happen because we retry forever
+
+                ensureCertificate(config.adminFqdn(), function (error, certFilePath, keyFilePath) {
+                    if (error) { // currently, this can never happen
+                        debug('Error obtaining certificate. Proceed anyway', error);
+                        return callback();
+                    }
+
+                    nginx.configureAdmin(certFilePath, keyFilePath, callback);
+                });
+            });
+        });
+    });
+}
+
+function isExpiringSync(certFilePath, hours) {
+    assert.strictEqual(typeof certFilePath, 'string');
+    assert.strictEqual(typeof hours, 'number');
+
+    if (!fs.existsSync(certFilePath)) return 2; // not found
+
+    var result = safe.child_process.spawnSync('/usr/bin/openssl', [ 'x509', '-checkend', String(60 * 60 * hours), '-in', certFilePath ]);
+
+    debug('isExpiringSync: %s %s %s', certFilePath, result.stdout.toString('utf8').trim(), result.status);
+
+    return result.status === 1; // 1 - expired 0 - not expired
+}
+
+function autoRenew(callback) {
+    debug('autoRenew: Checking certificates for renewal');
+    callback = callback || NOOP_CALLBACK;
+
+    apps.getAll(function (error, allApps) {
+        if (error) return callback(error);
+
+        allApps.push({ location: 'my' }); // inject fake webadmin app
+
+        var expiringApps = [ ];
+        for (var i = 0; i < allApps.length; i++) {
+            var appDomain = config.appFqdn(allApps[i].location);
+            var certFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.cert');
+            var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
+
+            if (!safe.fs.existsSync(keyFilePath)) {
+                debug('autoRenew: no existing key file for %s. skipping', appDomain);
+                continue;
+            }
+
+            if (isExpiringSync(certFilePath, 24 * 30)) { // expired or not found
+                expiringApps.push(allApps[i]);
+            }
+        }
+
+        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return config.appFqdn(a.location); }));
+
+        getApi(function (error, api, apiOptions) {
+            if (error) return callback(error);
+
+            async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
+                var domain = config.appFqdn(app.location);
+                debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+
+                api.getCertificate(domain, apiOptions, function (error) {
+                    var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+                    var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+                    mailer.certificateRenewed(domain, error ? error.message : '');
+
+                    if (error) {
+                        debug('autoRenew: could not renew cert for %s because %s', domain, error);
+
+                        // check if we should fallback if we expire in the coming day
+                        if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
+
+                        debug('autoRenew: using fallback certs for %s since it expires soon', domain, error);
+
+                        certFilePath = 'cert/host.cert';
+                        keyFilePath = 'cert/host.key';
+                    } else {
+                        debug('autoRenew: certificate for %s renewed', domain);
+                    }
+
+                    // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
+                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                        nginx.configureAdmin.bind(null, certFilePath, keyFilePath)
+                        : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
+
+                    configureFunc(function (ignoredError) {
+                        if (ignoredError) debug('fallbackExpiredCertificates: error reconfiguring app', ignoredError);
+
+                        iteratorCallback(); // move to next app
+                    });
+                });
+            });
+        });
+    });
+}
+
+// note: https://tools.ietf.org/html/rfc4346#section-7.4.2 (certificate_list) requires that the
+// servers certificate appears first (and not the intermediate cert)
+function validateCertificate(cert, key, fqdn) {
+    assert(cert === null || typeof cert === 'string');
+    assert(key === null || typeof key === 'string');
+    assert.strictEqual(typeof fqdn, 'string');
+
+    if (cert === null && key === null) return null;
+    if (!cert && key) return new Error('missing cert');
+    if (cert && !key) return new Error('missing key');
+
+    var content;
+    try {
+        content = x509.parseCert(cert);
+    } catch (e) {
+        return new Error('invalid cert: ' + e.message);
+    }
+
+    // check expiration
+    if (content.notAfter < new Date()) return new Error('cert expired');
+
+    function matchesDomain(domain) {
+        if (domain === fqdn) return true;
+        if (domain.indexOf('*') === 0 && domain.slice(2) === fqdn.slice(fqdn.indexOf('.') + 1)) return true;
+
+        return false;
+    }
+
+    // check domain
+    var domains = content.altNames.concat(content.subject.commonName);
+    if (!domains.some(matchesDomain)) return new Error(util.format('cert is not valid for this domain. Expecting %s in %j', fqdn, domains));
+
+    // http://httpd.apache.org/docs/2.0/ssl/ssl_faq.html#verify
+    var certModulus = safe.child_process.execSync('openssl x509 -noout -modulus', { encoding: 'utf8', input: cert });
+    var keyModulus = safe.child_process.execSync('openssl rsa -noout -modulus', { encoding: 'utf8', input: key });
+    if (certModulus !== keyModulus) return new Error('key does not match the cert');
+
+    return null;
+}
+
+function setFallbackCertificate(cert, key, callback) {
+    assert.strictEqual(typeof cert, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateCertificate(cert, key, '*.' + config.fqdn());
+    if (error) return callback(new CertificatesError(CertificatesError.INVALID_CERT, error.message));
+
+    // backup the cert
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.cert'), cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, 'host.key'), key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+
+    // copy over fallback cert
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+
+    nginx.reload(function (error) {
+        if (error) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function setAdminCertificate(cert, key, callback) {
+    assert.strictEqual(typeof cert, 'string');
+    assert.strictEqual(typeof key, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var vhost = config.appFqdn(constants.ADMIN_LOCATION);
+    var certFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.cert');
+    var keyFilePath = path.join(paths.APP_CERTS_DIR, vhost + '.key');
+
+    var error = validateCertificate(cert, key, vhost);
+    if (error) return callback(new CertificatesError(CertificatesError.INVALID_CERT, error.message));
+
+    // backup the cert
+    if (!safe.fs.writeFileSync(certFilePath, cert)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+    if (!safe.fs.writeFileSync(keyFilePath, key)) return callback(new CertificatesError(CertificatesError.INTERNAL_ERROR, safe.error.message));
+
+    nginx.configureAdmin(certFilePath, keyFilePath, callback);
+}
+
+function ensureCertificate(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // check if user uploaded a specific cert. ideally, we should not mix user certs and automatic certs as we do here...
+    var userCertFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
+    var userKeyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
+
+    if (fs.existsSync(userCertFilePath) && fs.existsSync(userKeyFilePath)) {
+        debug('ensureCertificate: %s. certificate already exists at %s', domain, userKeyFilePath);
+
+        if (!isExpiringSync(userCertFilePath, 24 * 1)) return callback(null, userCertFilePath, userKeyFilePath);
+    }
+
+    debug('ensureCertificate: %s cert require renewal', domain);
+
+    getApi(function (error, api, apiOptions) {
+        if (error) return callback(error);
+
+        debug('ensureCertificate: getting certificate for %s with options %j', domain, apiOptions);
+
+        api.getCertificate(domain, apiOptions, function (error, certFilePath, keyFilePath) {
+            if (error) {
+                debug('ensureCertificate: could not get certificate. using fallback certs', error);
+                return callback(null, 'cert/host.cert', 'cert/host.key'); // use fallback certs
+            }
+
+            callback(null, certFilePath, keyFilePath);
+        });
+    });
+}

src/clientdb.js

@@ -8,19 +8,27 @@ exports = module.exports = {
     getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
-    update: update,
     getByAppId: getByAppId,
-    delByAppId: delByAppId,
+    getByAppIdAndType: getByAppIdAndType,
 
-    _clear: clear
+    delByAppId: delByAppId,
+    delByAppIdAndType: delByAppIdAndType,
+
+    _clear: clear,
+
+    TYPE_EXTERNAL: 'external',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy',
+    TYPE_ADMIN: 'admin'
 };
 
 var assert = require('assert'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror.js');
 
-var CLIENTS_FIELDS = [ 'id', 'appId', 'clientSecret', 'redirectURI', 'scope' ].join(',');
+var CLIENTS_FIELDS = [ 'id', 'appId', 'type', 'clientSecret', 'redirectURI', 'scope' ].join(',');
-var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
+var CLIENTS_FIELDS_PREFIXED = [ 'clients.id', 'clients.appId', 'clients.type', 'clients.clientSecret', 'clients.redirectURI', 'clients.scope' ].join(',');
 
 function get(id, callback) {
     assert.strictEqual(typeof id, 'string');
@@ -67,37 +75,33 @@ function getByAppId(appId, callback) {
     });
 }
 
-function add(id, appId, clientSecret, redirectURI, scope, callback) {
+function getByAppIdAndType(appId, type, callback) {
-    assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof clientSecret, 'string');
+    assert.strictEqual(typeof type, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var data = [ id, appId, clientSecret, redirectURI, scope ];
+    database.query('SELECT ' + CLIENTS_FIELDS + ' FROM clients WHERE appId = ? AND type = ? LIMIT 1', [ appId, type ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-    database.query('INSERT INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?)', data, function (error, result) {
+        return callback(null, result[0]);
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-
-        callback(null);
     });
 }
 
-function update(id, appId, clientSecret, redirectURI, scope, callback) {
+function add(id, appId, type, clientSecret, redirectURI, scope, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof clientSecret, 'string');
     assert.strictEqual(typeof redirectURI, 'string');
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var data = [ appId, clientSecret, redirectURI, scope, id ];
+    var data = [ id, appId, type, clientSecret, redirectURI, scope ];
 
-    database.query('UPDATE clients SET appId = ?, clientSecret = ?, redirectURI = ?, scope = ? WHERE id = ?', data, function (error, result) {
+    database.query('INSERT INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
-        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (error || result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
     });
@@ -127,6 +131,19 @@ function delByAppId(appId, callback) {
     });
 }
 
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM clients WHERE appId=? AND type=?', [ appId, type ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 

src/clients.js

@@ -5,7 +5,6 @@ exports = module.exports = {
 
     add: add,
     get: get,
-    update: update,
     del: del,
     getAllWithDetailsByUserId: getAllWithDetailsByUserId,
     getClientTokensByUserId: getClientTokensByUserId,
@@ -43,6 +42,7 @@ function ClientsError(reason, errorOrMessage) {
 }
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
+ClientsError.INVALID_CLIENT = 'Invalid client';
 
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
@@ -55,8 +55,9 @@ function validateScope(scope) {
     return null;
 }
 
-function add(appIdentifier, redirectURI, scope, callback) {
+function add(appId, type, redirectURI, scope, callback) {
-    assert.strictEqual(typeof appIdentifier, 'string');
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof redirectURI, 'string');
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -67,12 +68,13 @@ function add(appIdentifier, redirectURI, scope, callback) {
     var id = 'cid-' + uuid.v4();
     var clientSecret = hat(256);
 
-    clientdb.add(id, appIdentifier, clientSecret, redirectURI, scope, function (error) {
+    clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
         if (error) return callback(error);
 
         var client = {
             id: id,
-            appId: appIdentifier,
+            appId: appId,
+            type: type,
             clientSecret: clientSecret,
             redirectURI: redirectURI,
             scope: scope
@@ -92,23 +94,6 @@ function get(id, callback) {
     });
 }
 
-// we only allow appIdentifier and redirectURI to be updated
-function update(id, appIdentifier, redirectURI, callback) {
-    assert.strictEqual(typeof id, 'string');
-    assert.strictEqual(typeof appIdentifier, 'string');
-    assert.strictEqual(typeof redirectURI, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    clientdb.get(id, function (error, result) {
-        if (error) return callback(error);
-
-        clientdb.update(id, appIdentifier, result.clientSecret, redirectURI, result.scope, function (error, result) {
-            if (error) return callback(error);
-            callback(null, result);
-        });
-    });
-}
-
 function del(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -127,54 +112,29 @@ function getAllWithDetailsByUserId(userId, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
         if (error) return callback(error);
 
-        // We have several types of records here
-        //   1) webadmin has an app id of 'webadmin'
-        //   2) oauth proxy records are always the app id prefixed with 'proxy-'
-        //   3) addon oauth records for apps prefixed with 'addon-'
-        //   4) external app records prefixed with 'external-'
-        //   5) normal apps on the cloudron without a prefix
-
         var tmp = [];
         async.each(results, function (record, callback) {
-            if (record.appId === constants.ADMIN_CLIENT_ID) {
+            if (record.type === clientdb.TYPE_ADMIN) {
                 record.name = constants.ADMIN_NAME;
                 record.location = constants.ADMIN_LOCATION;
-                record.type = 'webadmin';
-
-                tmp.push(record);
-
-                return callback(null);
-            } else if (record.appId === constants.TEST_CLIENT_ID) {
-                record.name = constants.TEST_NAME;
-                record.location = constants.TEST_LOCATION;
-                record.type = 'test';
 
                 tmp.push(record);
 
                 return callback(null);
             }
 
-            var appId = record.appId;
+            appdb.get(record.appId, function (error, result) {
-            var type = 'app';
-
-            // Handle our different types of oauth clients
-            if (record.appId.indexOf('addon-') === 0) {
-                appId = record.appId.slice('addon-'.length);
-                type = 'addon';
-            } else if (record.appId.indexOf('proxy-') === 0) {
-                appId = record.appId.slice('proxy-'.length);
-                type = 'proxy';
-            }
-
-            appdb.get(appId, function (error, result) {
                 if (error) {
                     console.error('Failed to get app details for oauth client', result, error);
                     return callback(null);  // ignore error so we continue listing clients
                 }
 
-                record.name = result.manifest.title + (record.appId.indexOf('proxy-') === 0 ? 'OAuth Proxy' : '');
+                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
+                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+
                 record.location = result.location;
-                record.type = type;
 
                 tmp.push(record);
 

src/cloudron.js

@@ -11,15 +11,24 @@ exports = module.exports = {
     getConfig: getConfig,
     getStatus: getStatus,
 
-    setCertificate: setCertificate,
-
     sendHeartbeat: sendHeartbeat,
 
+    updateToLatest: updateToLatest,
     update: update,
     reboot: reboot,
-    migrate: migrate,
     backup: backup,
-    ensureBackup: ensureBackup};
+    retire: retire,
+    ensureBackup: ensureBackup,
+
+    isConfiguredSync: isConfiguredSync,
+
+    checkDiskSpace: checkDiskSpace,
+
+    events: new (require('events').EventEmitter)(),
+
+    EVENT_ACTIVATED: 'activated',
+    EVENT_CONFIGURED: 'configured'
+};
 
 var apps = require('./apps.js'),
     AppsError = require('./apps.js').AppsError,
@@ -30,15 +39,18 @@ var apps = require('./apps.js'),
     clientdb = require('./clientdb.js'),
     config = require('./config.js'),
     debug = require('debug')('box:cloudron'),
+    df = require('node-df'),
     fs = require('fs'),
     locker = require('./locker.js'),
+    mailer = require('./mailer.js'),
+    os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
     progress = require('./progress.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
-    SettingsError = settings.SettingsError,
     shell = require('./shell.js'),
+    subdomains = require('./subdomains.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     tokendb = require('./tokendb.js'),
@@ -46,16 +58,20 @@ var apps = require('./apps.js'),
     user = require('./user.js'),
     UserError = user.UserError,
     userdb = require('./userdb.js'),
-    util = require('util');
+    util = require('util'),
+    webhooks = require('./webhooks.js');
 
-var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
+var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
-    REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
     BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
     BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
-    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
+    INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
+    RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh');
 
-var gAddMailDnsRecordsTimerId = null,
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-    gCloudronDetails = null;            // cached cloudron details like region,size...
+
+var gUpdatingDns = false,                // flag for dns update reentrancy
+    gCloudronDetails = null,             // cached cloudron details like region,size...
+    gIsConfigured = null;                // cached configured state so that return value is synchronous. null means we are not initialized yet
 
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
@@ -73,7 +89,6 @@ function ignoreError(func) {
     };
 }
 
-
 function CloudronError(reason, errorOrMessage) {
     assert.strictEqual(typeof reason, 'string');
     assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -102,30 +117,67 @@ CloudronError.BAD_EMAIL = 'Bad email';
 CloudronError.BAD_PASSWORD = 'Bad password';
 CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
+CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
 
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (process.env.BOX_ENV !== 'test') {
+    exports.events.on(exports.EVENT_CONFIGURED, addDnsRecords);
-        addMailDnsRecords();
-    }
 
-    // Send heartbeat once we are up and running, this speeds up the Cloudron creation, as otherwise we are bound to the cron.js settings
+    syncConfigState(callback);
-    sendHeartbeat();
-
-    callback(null);
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clearTimeout(gAddMailDnsRecordsTimerId);
+    exports.events.removeListener(exports.EVENT_CONFIGURED, addDnsRecords);
-    gAddMailDnsRecordsTimerId = null;
 
     callback(null);
 }
 
+function isConfiguredSync() {
+    return gIsConfigured === true;
+}
+
+function isConfigured(callback) {
+    // set of rules to see if we have the configs required for cloudron to function
+    // note this checks for missing configs and not invalid configs
+
+    settings.getDnsConfig(function (error, dnsConfig) {
+        if (error) return callback(error);
+
+        if (!dnsConfig) return callback(null, false);
+
+        var isConfigured = (config.isCustomDomain() && dnsConfig.provider === 'route53') ||
+                        (!config.isCustomDomain() && dnsConfig.provider === 'caas');
+
+        callback(null, isConfigured);
+    });
+}
+
+function syncConfigState(callback) {
+    assert(!gIsConfigured);
+
+    callback = callback || NOOP_CALLBACK;
+
+    isConfigured(function (error, configured) {
+        if (error) return callback(error);
+
+        debug('syncConfigState: configured = %s', configured);
+
+        if (configured) {
+            exports.events.emit(exports.EVENT_CONFIGURED);
+        } else {
+            settings.events.once(settings.DNS_CONFIG_KEY, function () { syncConfigState(); }); // check again later
+        }
+
+        gIsConfigured = configured;
+
+        callback();
+    });
+}
+
 function setTimeZone(ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -133,13 +185,13 @@ function setTimeZone(ip, callback) {
     debug('setTimeZone ip:%s', ip);
 
     superagent.get('http://www.telize.com/geoip/' + ip).end(function (error, result) {
-        if (error || result.statusCode !== 200) {
+        if ((error && !error.response) || result.statusCode !== 200) {
-            debug('Failed to get geo location', error);
+            debug('Failed to get geo location: %s', error.message);
             return callback(null);
         }
 
         if (!result.body.timezone) {
-            debug('No timezone in geoip response');
+            debug('No timezone in geoip response : %j', result.body);
             return callback(null);
         }
 
@@ -149,32 +201,26 @@ function setTimeZone(ip, callback) {
     });
 }
 
-function activate(username, password, email, name, ip, callback) {
+function activate(username, password, email, displayName, ip, callback) {
     assert.strictEqual(typeof username, 'string');
     assert.strictEqual(typeof password, 'string');
     assert.strictEqual(typeof email, 'string');
+    assert.strictEqual(typeof displayName, 'string');
     assert.strictEqual(typeof ip, 'string');
-    assert(!name || typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     debug('activating user:%s email:%s', username, email);
 
-    setTimeZone(ip, function () { });
+    setTimeZone(ip, function () { }); // TODO: get this from user. note that timezone is detected based on the browser location and not the cloudron region
 
-    if (!name) name = settings.getDefaultSync(settings.CLOUDRON_NAME_KEY);
+    user.createOwner(username, password, email, displayName, function (error, userObject) {
-
-    settings.setCloudronName(name, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_NAME));
-        if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
-
-        user.createOwner(username, password, email, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
         if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
         if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
         if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-            clientdb.getByAppId('webadmin', function (error, result) {
+        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             // Also generate a token so the admin creation can also act as a login
@@ -184,11 +230,13 @@ function activate(username, password, email, name, ip, callback) {
             tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
+                // EE API is sync. do not keep the REST API reponse waiting
+                process.nextTick(function () { exports.events.emit(exports.EVENT_ACTIVATED); });
+
                 callback(null, { token: token, expires: expires });
             });
         });
     });
-    });
 }
 
 function getStatus(callback) {
@@ -203,6 +251,9 @@ function getStatus(callback) {
             callback(null, {
                 activated: count !== 0,
                 version: config.version(),
+                boxVersionsUrl: config.get('boxVersionsUrl'),
+                apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool
+                provider: config.provider(),
                 cloudronName: cloudronName
             });
         });
@@ -214,12 +265,21 @@ function getCloudronDetails(callback) {
 
     if (gCloudronDetails) return callback(null, gCloudronDetails);
 
+    if (!config.token()) {
+        gCloudronDetails = {
+            region: null,
+            size: null
+        };
+
+        return callback(null, gCloudronDetails);
+    }
+
     superagent
         .get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn())
         .query({ token: config.token() })
         .end(function (error, result) {
-            if (error) return callback(error);
+            if (error && !error.response) return callback(error);
-            if (result.status !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            if (result.statusCode !== 200) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
 
             gCloudronDetails = result.body.box;
 
@@ -230,10 +290,9 @@ function getCloudronDetails(callback) {
 function getConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    // TODO avoid pyramid of awesomeness with async
     getCloudronDetails(function (error, result) {
         if (error) {
-            console.error('Failed to fetch cloudron details.', error);
+            debug('Failed to fetch cloudron details.', error);
 
             // set fallback values to avoid dependency on appstore
             result = {
@@ -248,12 +307,15 @@ function getConfig(callback) {
             settings.getDeveloperMode(function (error, developerMode) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
+                sysinfo.getIp(function (error, ip) {
+                    if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
+
                     callback(null, {
                         apiServerOrigin: config.apiServerOrigin(),
                         webServerOrigin: config.webServerOrigin(),
                         isDev: config.isDev(),
                         fqdn: config.fqdn(),
-                    ip: sysinfo.getIp(),
+                        ip: ip,
                         version: config.version(),
                         update: updateChecker.getUpdateInfo(),
                         progress: progress.get(),
@@ -261,102 +323,137 @@ function getConfig(callback) {
                         developerMode: developerMode,
                         region: result.region,
                         size: result.size,
+                        memory: os.totalmem(),
+                        provider: config.provider(),
                         cloudronName: cloudronName
                     });
                 });
             });
         });
+    });
 }
 
 function sendHeartbeat() {
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
+    if (!config.token()) return;
 
-    // TODO: this must be a POST
+    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
-    superagent.get(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
+    superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(10000).end(function (error, result) {
-        if (error) debug('Error sending heartbeat.', error);
+        if (error && !error.response) debug('Network error sending heartbeat.', error);
         else if (result.statusCode !== 200) debug('Server responded to heartbeat with %s %s', result.statusCode, result.text);
         else debug('Heartbeat sent to %s', url);
     });
 }
 
-function sendMailDnsRecordsRequest(callback) {
+function readDkimPublicKeySync() {
-    assert.strictEqual(typeof callback, 'function');
-
-    var DKIM_SELECTOR = 'mail';
-    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
-
     var dkimPublicKeyFile = path.join(paths.MAIL_DATA_DIR, 'dkim/' + config.fqdn() + '/public');
     var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');
 
-    if (publicKey === null) return callback(new Error('Error reading dkim public key'));
+    if (publicKey === null) {
+        debug('Error reading dkim public key.', safe.error);
+        return null;
+    }
 
     // remove header, footer and new lines
     publicKey = publicKey.split('\n').slice(1, -2).join('');
 
-    // note that dmarc requires special DNS records for external RUF and RUA
+    return publicKey;
-    var records = [
-        // softfail all mails not from our IP. Note that this uses IP instead of 'a' should we use a load balancer in the future
-        { subdomain: '', type: 'TXT', value: '"v=spf1 ip4:' + sysinfo.getIp() + ' ~all"' },
-        // t=s limits the domainkey to this domain and not it's subdomains
-        { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', value: '"v=DKIM1; t=s; p=' + publicKey + '"' },
-        // DMARC requires special setup if report email id is in different domain
-        { subdomain: '_dmarc', type: 'TXT', value: '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' }
-    ];
-
-    debug('sendMailDnsRecords request:%s', JSON.stringify(records));
-
-    superagent
-        .post(config.apiServerOrigin() + '/api/v1/subdomains')
-        .set('Accept', 'application/json')
-        .query({ token: config.token() })
-        .send({ records: records })
-        .end(function (error, res) {
-            if (error) return callback(error);
-
-            debug('sendMailDnsRecords status: %s', res.status);
-
-            if (res.status === 409) return callback(null); // already registered
-
-            if (res.status !== 201) return callback(new Error(util.format('Failed to add Mail DNS records: %s %j', res.status, res.body)));
-
-            return callback(null, res.body.ids);
-        });
 }
 
-function addMailDnsRecords() {
+function txtRecordsWithSpf(callback) {
-    if (config.get('mailDnsRecordIds').length !== 0) return; // already registered
-
-    sendMailDnsRecordsRequest(function (error, ids) {
-        if (error) {
-            console.error('Mail DNS record addition failed', error);
-            gAddMailDnsRecordsTimerId = setTimeout(addMailDnsRecords, 30000);
-            return;
-        }
-
-        debug('Added Mail DNS records successfully');
-        config.set('mailDnsRecordIds', ids);
-    });
-}
-
-function setCertificate(certificate, key, callback) {
-    assert.strictEqual(typeof certificate, 'string');
-    assert.strictEqual(typeof key, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('Updating certificates');
+    subdomains.get('', 'TXT', function (error, txtRecords) {
+        if (error) return callback(error);
 
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), certificate)) {
+        debug('txtRecordsWithSpf: current txt records - %j', txtRecords);
-        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+
+        var i, validSpf;
+
+        for (i = 0; i < txtRecords.length; i++) {
+            if (txtRecords[i].indexOf('"v=spf1 ') !== 0) continue; // not SPF
+
+            validSpf = txtRecords[i].indexOf(' a:' + config.adminFqdn() + ' ') !== -1;
+            break;
         }
 
-    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), key)) {
+        if (validSpf) return callback(null, null);
-        return callback(new CloudronError(CloudronError.INTERNAL_ERROR, safe.error.message));
+
+        if (i == txtRecords.length) {
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ~all"';
+        } else {
+            txtRecords[i] = '"v=spf1 a:' + config.adminFqdn() + ' ' + txtRecords[i].slice('"v=spf1 '.length);
         }
 
-    shell.sudo('setCertificate', [ RELOAD_NGINX_CMD ], function (error) {
+        return callback(null, txtRecords);
+    });
+}
+
+function addDnsRecords() {
+    var callback = NOOP_CALLBACK;
+
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    if (gUpdatingDns) {
+        debug('addDnsRecords: dns update already in progress');
+        return callback();
+    }
+    gUpdatingDns = true;
+
+    var DKIM_SELECTOR = 'cloudron';
+    var DMARC_REPORT_EMAIL = 'dmarc-report@cloudron.io';
+
+    var dkimKey = readDkimPublicKeySync();
+    if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('internal error failed to read dkim public key')));
+
+    sysinfo.getIp(function (error, ip) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        return callback(null);
+        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
+        // t=s limits the domainkey to this domain and not it's subdomains
+        var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
+        // DMARC requires special setup if report email id is in different domain
+        var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
+
+        var records = [ ];
+        if (config.isCustomDomain()) {
+            records.push(webadminRecord);
+            records.push(dkimRecord);
+        } else {
+            // for custom domains, we show a nakeddomain.html page
+            var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
+
+            records.push(nakedDomainRecord);
+            records.push(webadminRecord);
+            records.push(dkimRecord);
+            records.push(dmarcRecord);
+        }
+
+        debug('addDnsRecords: %j', records);
+
+        async.retry({ times: 10, interval: 20000 }, function (retryCallback) {
+            txtRecordsWithSpf(function (error, txtRecords) {
+                if (error) return retryCallback(error);
+
+                if (txtRecords) records.push({ subdomain: '', type: 'TXT', values: txtRecords });
+
+                debug('addDnsRecords: will update %j', records);
+
+                async.mapSeries(records, function (record, iteratorCallback) {
+                    subdomains.update(record.subdomain, record.type, record.values, iteratorCallback);
+                }, function (error, changeIds) {
+                    if (error) debug('addDnsRecords: failed to update : %s. will retry', error);
+                    else debug('addDnsRecords: records %j added with changeIds %j', records, changeIds);
+
+                    retryCallback(error);
+                });
+            });
+        }, function (error) {
+            gUpdatingDns = false;
+
+            debug('addDnsRecords: done updating records with error:', error);
+
+            callback(error);
+        });
     });
 }
 
@@ -364,49 +461,6 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function migrate(size, region, callback) {
-    assert.strictEqual(typeof size, 'string');
-    assert.strictEqual(typeof region, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var error = locker.lock(locker.OP_MIGRATE);
-    if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
-
-    function unlock(error) {
-        if (error) {
-            debug('Failed to migrate', error);
-            locker.unlock(locker.OP_MIGRATE);
-        } else {
-            debug('Migration initiated successfully');
-            // do not unlock; cloudron is migrating
-        }
-
-        return;
-    }
-
-    // initiate the migration in the background
-    backupBoxAndApps(function (error, restoreKey) {
-        if (error) return unlock(error);
-
-        debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
-
-        superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send({ size: size, region: region, restoreKey: restoreKey })
-          .end(function (error, result) {
-            if (error) return unlock(error);
-            if (result.status === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.status === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.status !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
-
-            return unlock(null);
-        });
-    });
-
-    callback(null);
-}
-
 function update(boxUpdateInfo, callback) {
     assert.strictEqual(typeof boxUpdateInfo, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -416,12 +470,20 @@ function update(boxUpdateInfo, callback) {
     var error = locker.lock(locker.OP_BOX_UPDATE);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    // ensure tools can 'wait' on progress
+    progress.set(progress.UPDATE, 0, 'Starting');
+
     // initiate the update/upgrade but do not wait for it
-    if (boxUpdateInfo.upgrade) {
+    if (config.version().match(/[-+]/) !== null && config.version().replace(/[-+].*/, '') === boxUpdateInfo.version) {
+        doShortCircuitUpdate(boxUpdateInfo, function (error) {
+            if (error) debug('Short-circuit update failed', error);
+            locker.unlock(locker.OP_BOX_UPDATE);
+        });
+    } else if (boxUpdateInfo.upgrade) {
         debug('Starting upgrade');
         doUpgrade(boxUpdateInfo, function (error) {
             if (error) {
-                debug('Upgrade failed with error: %s', error);
+                console.error('Upgrade failed with error:', error);
                 locker.unlock(locker.OP_BOX_UPDATE);
             }
         });
@@ -429,7 +491,7 @@ function update(boxUpdateInfo, callback) {
         debug('Starting update');
         doUpdate(boxUpdateInfo, function (error) {
             if (error) {
-                debug('Update failed with error: %s', error);
+                console.error('Update failed with error:', error);
                 locker.unlock(locker.OP_BOX_UPDATE);
             }
         });
@@ -438,6 +500,26 @@ function update(boxUpdateInfo, callback) {
     callback(null);
 }
 
+
+function updateToLatest(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var boxUpdateInfo = updateChecker.getUpdateInfo().box;
+    if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
+
+    update(boxUpdateInfo, callback);
+}
+
+function doShortCircuitUpdate(boxUpdateInfo, callback) {
+    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
+
+    debug('Starting short-circuit from prerelease version %s to release version %s', config.version(), boxUpdateInfo.version);
+    config.setVersion(boxUpdateInfo.version);
+    progress.clear(progress.UPDATE);
+    updateChecker.resetUpdateInfo();
+    callback();
+}
+
 function doUpgrade(boxUpdateInfo, callback) {
     assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
 
@@ -446,7 +528,7 @@ function doUpgrade(boxUpdateInfo, callback) {
         callback(e);
     }
 
-    progress.set(progress.UPDATE, 5, 'Create app and box backup for upgrade');
+    progress.set(progress.UPDATE, 5, 'Backing up for upgrade');
 
     backupBoxAndApps(function (error) {
         if (error) return upgradeError(error);
@@ -455,14 +537,14 @@ function doUpgrade(boxUpdateInfo, callback) {
           .query({ token: config.token() })
           .send({ version: boxUpdateInfo.version })
           .end(function (error, result) {
-            if (error) return upgradeError(new Error('Error making upgrade request: ' + error));
+            if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
-            if (result.status !== 202) return upgradeError(new Error('Server not ready to upgrade: ' + result.body));
+            if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
 
             progress.set(progress.UPDATE, 10, 'Updating base system');
 
             // no need to unlock since this is the last thing we ever do on this box
-
+            callback();
-            callback(null);
+            retire();
         });
     });
 }
@@ -475,50 +557,50 @@ function doUpdate(boxUpdateInfo, callback) {
         callback(e);
     }
 
-    progress.set(progress.UPDATE, 5, 'Create box backup for update');
+    progress.set(progress.UPDATE, 5, 'Backing up for update');
 
-    backupBox(function (error) {
+    backupBoxAndApps(function (error) {
         if (error) return updateError(error);
 
-        // fetch a signed sourceTarballUrl
-        superagent.get(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/sourcetarballurl')
-          .query({ token: config.token(), boxVersion: boxUpdateInfo.version })
-          .end(function (error, result) {
-            if (error) return updateError(new Error('Error fetching sourceTarballUrl: ' + error));
-            if (result.status !== 200) return updateError(new Error('Error fetching sourceTarballUrl status: ' + result.status));
-            if (!safe.query(result, 'body.url')) return updateError(new Error('Error fetching sourceTarballUrl response: ' + result.body));
-
         // NOTE: the args here are tied to the installer revision, box code and appstore provisioning logic
         var args = {
-                sourceTarballUrl: result.body.url,
+            sourceTarballUrl: boxUpdateInfo.sourceTarballUrl,
 
-                // IMPORTANT: if you change this, fix up argparser.sh as well. keep these sorted for readability
+            // this data is opaque to the installer
             data: {
-                    apiServerOrigin: config.apiServerOrigin(),
-                    boxVersionsUrl: config.get('boxVersionsUrl'),
-                    fqdn: config.fqdn(),
-                    isCustomDomain: config.isCustomDomain(),
-                    restoreKey: null,
-                    restoreUrl: null,
-                    tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
-                    tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
                 token: config.token(),
-                    version: boxUpdateInfo.version,
+                apiServerOrigin: config.apiServerOrigin(),
+                webServerOrigin: config.webServerOrigin(),
+                fqdn: config.fqdn(),
+                tlsCert: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.cert'), 'utf8'),
+                tlsKey: fs.readFileSync(path.join(paths.NGINX_CERT_DIR, 'host.key'), 'utf8'),
+                isCustomDomain: config.isCustomDomain(),
+
+                appstore: {
+                    token: config.token(),
+                    apiServerOrigin: config.apiServerOrigin()
+                },
+                caas: {
+                    token: config.token(),
+                    apiServerOrigin: config.apiServerOrigin(),
                     webServerOrigin: config.webServerOrigin()
+                },
+
+                version: boxUpdateInfo.version,
+                boxVersionsUrl: config.get('boxVersionsUrl')
             }
         };
 
         debug('updating box %j', args);
 
         superagent.post(INSTALLER_UPDATE_URL).send(args).end(function (error, result) {
-                if (error) return updateError(error);
+            if (error && !error.response) return updateError(error);
-                if (result.status !== 202) return updateError(new Error('Error initiating update: ' + result.body));
+            if (result.statusCode !== 202) return updateError(new Error('Error initiating update: ' + JSON.stringify(result.body)));
 
             progress.set(progress.UPDATE, 10, 'Updating cloudron software');
 
             callback(null);
         });
-        });
 
         // Do not add any code here. The installer script will stop the box code any instant
     });
@@ -530,6 +612,9 @@ function backup(callback) {
     var error = locker.lock(locker.OP_FULL_BACKUP);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    // ensure tools can 'wait' on progress
+    progress.set(progress.BACKUP, 0, 'Starting');
+
     // start the backup operation in the background
     backupBoxAndApps(function (error) {
         if (error) console.error('backup failed.', error);
@@ -541,9 +626,9 @@ function backup(callback) {
 }
 
 function ensureBackup(callback) {
-    callback = callback || function () { };
+    callback = callback || NOOP_CALLBACK;
 
-    backups.getAllPaged(1, 1, function (error, backups) {
+    backups.getPaged(1, 1, function (error, backups) {
         if (error) {
             debug('Unable to list backups', error);
             return callback(error); // no point trying to backup if appstore is down
@@ -561,7 +646,7 @@ function ensureBackup(callback) {
 function backupBoxWithAppBackupIds(appBackupIds, callback) {
     assert(util.isArray(appBackupIds));
 
-    backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
+    backups.getBackupUrl(appBackupIds, function (error, result) {
         if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
@@ -569,16 +654,19 @@ function backupBoxWithAppBackupIds(appBackupIds, callback) {
 
         async.series([
             ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
-            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
+            shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey, result.sessionToken ]),
             ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
         ], function (error) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             debug('backup: successful');
 
+            webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
+                if (error) return callback(error);
                 callback(null, result.id);
             });
         });
+    });
 }
 
 // this function expects you to have a lock
@@ -595,7 +683,7 @@ function backupBox(callback) {
 
 // this function expects you to have a lock
 function backupBoxAndApps(callback) {
-    callback = callback || function () { }; // callback can be empty for timer triggered backup
+    callback = callback || NOOP_CALLBACK;
 
     apps.getAll(function (error, allApps) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
@@ -609,14 +697,14 @@ function backupBoxAndApps(callback) {
             ++processed;
 
             apps.backupApp(app, app.manifest.addons, function (error, backupId) {
-                progress.set(progress.BACKUP, step * processed, 'Backing up app at ' + app.location);
+                if (error && error.reason !== AppsError.BAD_STATE) {
-
+                    debugApp(app, 'Unable to backup', error);
-                if (error && error.reason === AppsError.BAD_STATE) {
+                    return iteratorCallback(error);
-                    debugApp(app, 'Skipping backup (istate:%s health:%s). using lastBackupId:%s', app.installationState, app.health, app.lastBackupId);
-                    backupId = app.lastBackupId;
                 }
 
-                return iteratorCallback(null, backupId);
+                progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
+
+                iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
             });
         }, function appsBackedUp(error, backupIds) {
             if (error) {
@@ -624,12 +712,49 @@ function backupBoxAndApps(callback) {
                 return callback(error);
             }
 
-            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
+            backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
 
-            backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
+            backupBoxWithAppBackupIds(backupIds, function (error, filename) {
                 progress.set(progress.BACKUP, 100, error ? error.message : '');
-                callback(error, restoreKey);
+
+                callback(error, filename);
             });
         });
     });
 }
+
+function checkDiskSpace(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Checking disk space');
+
+    df(function (error, entries) {
+        if (error) {
+            debug('df error %s', error.message);
+            mailer.outOfDiskSpace(error.message);
+            return callback();
+        }
+
+        var oos = entries.some(function (entry) {
+            return (entry.mount === paths.DATA_DIR && entry.capacity >= 0.90) ||
+                   (entry.mount === '/' && entry.used <= (1.25 * 1024 * 1024)); // 1.5G
+        });
+
+        debug('Disk space checked. ok: %s', !oos);
+
+        if (oos) mailer.outOfDiskSpace(JSON.stringify(entries, null, 4));
+
+        callback();
+    });
+}
+
+function retire(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    var data = {
+        isCustomDomain: config.isCustomDomain(),
+        fqdn: config.fqdn()
+    };
+    shell.sudo('retire', [ RETIRE_CMD, JSON.stringify(data) ], callback);
+}
+

src/collectd.config.ejs

@@ -1,6 +1,6 @@
 LoadPlugin "table"
 <Plugin table>
-  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.stat">
+  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.stat">
     Instance "<%= appId %>-memory"
     Separator " \\n"
     <Result>
@@ -10,7 +10,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/memory/system.slice/docker-<%= containerId %>.scope/memory.max_usage_in_bytes">
+  <Table "/sys/fs/cgroup/memory/system.slice/docker.service/docker/<%= containerId %>/memory.max_usage_in_bytes">
     Instance "<%= appId %>-memory"
     Separator "\\n"
     <Result>
@@ -20,7 +20,7 @@ LoadPlugin "table"
     </Result>
   </Table>
 
-  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker-<%= containerId %>.scope/cpuacct.stat">
+  <Table "/sys/fs/cgroup/cpuacct/system.slice/docker/<%= containerId %>/cpuacct.stat">
     Instance "<%= appId %>-cpu"
     Separator " \\n"
     <Result>

src/config.js

@@ -4,6 +4,8 @@
 
 exports = module.exports = {
     baseDir: baseDir,
+    dnsInSync: dnsInSync,
+    setDnsInSync: setDnsInSync,
 
     // values set here will be lost after a upgrade/update. use the sqlite database
     // for persistent values that need to be backed up
@@ -15,23 +17,28 @@ exports = module.exports = {
     TEST: process.env.BOX_ENV === 'test',
 
     // convenience getters
+    provider: provider,
     apiServerOrigin: apiServerOrigin,
     webServerOrigin: webServerOrigin,
     fqdn: fqdn,
     token: token,
     version: version,
+    setVersion: setVersion,
     isCustomDomain: isCustomDomain,
     database: database,
 
     // these values are derived
     adminOrigin: adminOrigin,
+    internalAdminOrigin: internalAdminOrigin,
+    adminFqdn: adminFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
+    adminEmail: adminEmail,
 
     isDev: isDev,
 
     // for testing resets to defaults
-    _reset: initConfig
+    _reset: _reset
 };
 
 var assert = require('assert'),
@@ -52,24 +59,41 @@ function baseDir() {
 
 var cloudronConfigFileName = path.join(baseDir(), 'configs/cloudron.conf');
 
+function dnsInSync() {
+    return !!safe.fs.statSync(require('./paths.js').DNS_IN_SYNC_FILE);
+}
+
+function setDnsInSync(content) {
+    safe.fs.writeFileSync(require('./paths.js').DNS_IN_SYNC_FILE, content || 'if this file exists, dns is in sync');
+}
+
 function saveSync() {
     fs.writeFileSync(cloudronConfigFileName, JSON.stringify(data, null, 4)); // functions are ignored by JSON.stringify
 }
 
+function _reset (callback) {
+    safe.fs.unlinkSync(cloudronConfigFileName);
+
+    initConfig();
+
+    if (callback) callback();
+}
+
 function initConfig() {
     // setup defaults
     data.fqdn = 'localhost';
 
     data.token = null;
-    data.mailServer = null;
     data.adminEmail = null;
-    data.mailDnsRecordIds = [ ];
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
     data.internalPort = 3001;
     data.ldapPort = 3002;
+    data.oauthProxyPort = 3003;
+    data.simpleAuthPort = 3004;
+    data.provider = 'caas';
 
     if (exports.CLOUDRON) {
         data.port = 3000;
@@ -86,6 +110,7 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
+        data.adminEmail = 'test@cloudron.foo';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -99,6 +124,9 @@ function initConfig() {
     saveSync();
 }
 
+// cleanup any old config file we have for tests
+if (exports.TEST) safe.fs.unlinkSync(cloudronConfigFileName);
+
 initConfig();
 
 // set(obj) or set(key, value)
@@ -121,6 +149,10 @@ function get(key) {
     return safe.query(data, key);
 }
 
+function adminEmail() {
+    return '"Cloudron" ' + get('adminEmail');
+}
+
 function apiServerOrigin() {
     return get('apiServerOrigin');
 }
@@ -141,10 +173,18 @@ function appFqdn(location) {
     return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
 }
 
+function adminFqdn() {
+    return appFqdn(constants.ADMIN_LOCATION);
+}
+
 function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }
 
+function internalAdminOrigin() {
+    return 'http://127.0.0.1:' + get('port');
+}
+
 function token() {
     return get('token');
 }
@@ -153,6 +193,10 @@ function version() {
     return get('version');
 }
 
+function setVersion(version) {
+    set('version', version);
+}
+
 function isCustomDomain() {
     return get('isCustomDomain');
 }
@@ -172,3 +216,6 @@ function isDev() {
     return /dev/i.test(get('boxVersionsUrl'));
 }
 
+function provider() {
+    return get('provider');
+}

src/constants.js

@@ -9,8 +9,6 @@ exports = module.exports = {
     ADMIN_CLIENT_ID: 'webadmin', // oauth client id
     ADMIN_APPID: 'admin', // admin appid (settingsdb)
 
-    TEST_NAME: 'Test',
+    DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024) // see also client.js
-    TEST_LOCATION: '',
-    TEST_CLIENT_ID: 'test'
 };
 

src/cron.js

@@ -7,9 +7,13 @@ exports = module.exports = {
 
 var apps = require('./apps.js'),
     assert = require('assert'),
+    certificates = require('./certificates.js'),
     cloudron = require('./cloudron.js'),
+    config = require('./config.js'),
     CronJob = require('cron').CronJob,
     debug = require('debug')('box:cron'),
+    janitor = require('./janitor.js'),
+    scheduler = require('./scheduler.js'),
     settings = require('./settings.js'),
     updateChecker = require('./updatechecker.js');
 
@@ -17,9 +21,12 @@ var gAutoupdaterJob = null,
     gBoxUpdateCheckerJob = null,
     gAppUpdateCheckerJob = null,
     gHeartbeatJob = null,
-    gBackupJob = null;
+    gBackupJob = null,
-
+    gCleanupTokensJob = null,
-var gInitialized = false;
+    gDockerVolumeCleanerJob = null,
+    gSchedulerSyncJob = null,
+    gCertificateRenewJob = null,
+    gCheckDiskSpaceJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 
@@ -34,27 +41,26 @@ var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (gInitialized) return callback();
+    gHeartbeatJob = new CronJob({
-
+        cronTime: '00 */1 * * * *', // every minute
-    settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
+        onTick: cloudron.sendHeartbeat,
-    settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
+        start: true
-
+    });
-    gInitialized = true;
+    cloudron.sendHeartbeat(); // latest unpublished version of CronJob has runOnInit
 
+    if (cloudron.isConfiguredSync()) {
         recreateJobs(callback);
+    } else {
+        cloudron.events.on(cloudron.EVENT_ACTIVATED, recreateJobs);
+        callback();
+    }
 }
 
 function recreateJobs(unusedTimeZone, callback) {
     if (typeof unusedTimeZone === 'function') callback = unusedTimeZone;
 
     settings.getAll(function (error, allSettings) {
-        if (gHeartbeatJob) gHeartbeatJob.stop();
+        debug('Creating jobs with timezone %s', allSettings[settings.TIME_ZONE_KEY]);
-        gHeartbeatJob = new CronJob({
-            cronTime: '00 */1 * * * *', // every minute
-            onTick: cloudron.sendHeartbeat,
-            start: true,
-            timeZone: allSettings[settings.TIME_ZONE_KEY]
-        });
 
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
@@ -64,6 +70,14 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCheckDiskSpaceJob) gCheckDiskSpaceJob.stop();
+        gCheckDiskSpaceJob = new CronJob({
+            cronTime: '00 30 */4 * * *', // every 4 hours
+            onTick: cloudron.checkDiskSpace,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
         if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
         gBoxUpdateCheckerJob = new CronJob({
             cronTime: '00 */10 * * * *', // every 10 minutes
@@ -80,14 +94,52 @@ function recreateJobs(unusedTimeZone, callback) {
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
 
+        if (gCleanupTokensJob) gCleanupTokensJob.stop();
+        gCleanupTokensJob = new CronJob({
+            cronTime: '00 */30 * * * *', // every 30 minutes
+            onTick: janitor.cleanupTokens,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
+        gDockerVolumeCleanerJob = new CronJob({
+            cronTime: '00 00 */12 * * *', // every 12 hours
+            onTick: janitor.cleanupDockerVolumes,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
+        gSchedulerSyncJob = new CronJob({
+            cronTime: config.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute
+            onTick: scheduler.sync,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        if (gCertificateRenewJob) gCertificateRenewJob.stop();
+        gCertificateRenewJob = new CronJob({
+            cronTime: '00 00 */12 * * *', // every 12 hours
+            onTick: certificates.autoRenew,
+            start: true,
+            timeZone: allSettings[settings.TIME_ZONE_KEY]
+        });
+
+        settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
+        settings.events.on(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
         autoupdatePatternChanged(allSettings[settings.AUTOUPDATE_PATTERN_KEY]);
 
+        settings.events.removeListener(settings.TIME_ZONE_KEY, recreateJobs);
+        settings.events.on(settings.TIME_ZONE_KEY, recreateJobs);
+
         if (callback) callback();
     });
 }
 
 function autoupdatePatternChanged(pattern) {
     assert.strictEqual(typeof pattern, 'string');
+    assert(gBoxUpdateCheckerJob);
 
     debug('Auto update pattern changed to %s', pattern);
 
@@ -110,32 +162,44 @@ function autoupdatePatternChanged(pattern) {
             }
         },
         start: true,
-        timeZone: gBoxUpdateCheckerJob.cronTime.timeZone // hack
+        timeZone: gBoxUpdateCheckerJob.cronTime.zone // hack
     });
 }
 
 function uninitialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (!gInitialized) return callback();
+    cloudron.events.removeListener(cloudron.EVENT_ACTIVATED, recreateJobs);
+
+    settings.events.removeListener(settings.TIME_ZONE_KEY, recreateJobs);
+    settings.events.removeListener(settings.AUTOUPDATE_PATTERN_KEY, autoupdatePatternChanged);
 
     if (gAutoupdaterJob) gAutoupdaterJob.stop();
     gAutoupdaterJob = null;
 
-    gBoxUpdateCheckerJob.stop();
+    if (gBoxUpdateCheckerJob) gBoxUpdateCheckerJob.stop();
     gBoxUpdateCheckerJob = null;
 
-    gAppUpdateCheckerJob.stop();
+    if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
     gAppUpdateCheckerJob = null;
 
-    gHeartbeatJob.stop();
+    if (gHeartbeatJob) gHeartbeatJob.stop();
     gHeartbeatJob = null;
 
-    gBackupJob.stop();
+    if (gBackupJob) gBackupJob.stop();
     gBackupJob = null;
 
-    gInitialized = false;
+    if (gCleanupTokensJob) gCleanupTokensJob.stop();
+    gCleanupTokensJob = null;
+
+    if (gDockerVolumeCleanerJob) gDockerVolumeCleanerJob.stop();
+    gDockerVolumeCleanerJob = null;
+
+    if (gSchedulerSyncJob) gSchedulerSyncJob.stop();
+    gSchedulerSyncJob = null;
+
+    if (gCertificateRenewJob) gCertificateRenewJob.stop();
+    gCertificateRenewJob = null;
 
     callback();
 }
-

src/database.js

@@ -118,6 +118,7 @@ function clear(callback) {
         require('./authcodedb.js')._clear,
         require('./clientdb.js')._clear,
         require('./tokendb.js')._clear,
+        require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
         require('./settingsdb.js')._clear
     ], callback);
@@ -126,6 +127,8 @@ function clear(callback) {
 function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
+    if (gConnectionPool === null) return callback(new Error('No database connection pool.'));
+
     gConnectionPool.getConnection(function (error, connection) {
         if (error) return callback(error);
 

src/databaseerror.js

@@ -30,3 +30,4 @@ DatabaseError.INTERNAL_ERROR = 'Internal error';
 DatabaseError.ALREADY_EXISTS = 'Entry already exist';
 DatabaseError.NOT_FOUND = 'Record not found';
 DatabaseError.BAD_FIELD = 'Invalid field';
+DatabaseError.IN_USE = 'In Use';

src/developer.js

@@ -13,6 +13,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     config = require('./config.js'),
+    debug = require('debug')('box:developer'),
     tokendb = require('./tokendb.js'),
     settings = require('./settings.js'),
     superagent = require('superagent'),
@@ -38,6 +39,7 @@ function DeveloperError(reason, errorOrMessage) {
 }
 util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
+DeveloperError.EXTERNAL_ERROR = 'External Error';
 
 function enabled(callback) {
     assert.strictEqual(typeof callback, 'function');
@@ -65,7 +67,7 @@ function issueDeveloperToken(user, callback) {
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'apps,settings,roleDeveloper', function (error) {
+    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users', function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
         callback(null, { token: token, expiresAt: expiresAt });
@@ -77,8 +79,12 @@ function getNonApprovedApps(callback) {
 
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
     superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).end(function (error, result) {
-        if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
+        if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
-        if (result.status !== 200) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
+        if (result.statusCode === 401) {
+            debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);
+            return callback(null, []);
+        }
+        if (result.statusCode !== 200) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
 
         callback(null, result.body.apps || []);
     });

src/digitalocean.js

@@ -1,46 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-exports = module.exports = {
-    checkPtrRecord: checkPtrRecord
-};
-
-var assert = require('assert'),
-    debug = require('debug')('box:digitalocean'),
-    dns = require('native-dns');
-
-function checkPtrRecord(ip, fqdn, callback) {
-    assert(ip === null || typeof ip === 'string');
-    assert.strictEqual(typeof fqdn, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('checkPtrRecord: ' + ip);
-
-    if (!ip) return callback(new Error('Network down'));
-
-    dns.resolve4('ns1.digitalocean.com', function (error, rdnsIps) {
-        if (error || rdnsIps.length === 0) return callback(new Error('Failed to query DO DNS'));
-
-        var reversedIp = ip.split('.').reverse().join('.');
-
-        var req = dns.Request({
-            question: dns.Question({ name: reversedIp + '.in-addr.arpa', type: 'PTR' }),
-            server: { address: rdnsIps[0] },
-            timeout: 5000
-        });
-
-        req.on('timeout', function () { return callback(new Error('Timedout')); });
-
-        req.on('message', function (error, message) {
-            if (error || !message.answer || message.answer.length === 0) return callback(new Error('Failed to query PTR'));
-
-            debug('checkPtrRecord: Actual:%s Expecting:%s', message.answer[0].data, fqdn);
-            callback(null, message.answer[0].data === fqdn);
-        });
-
-        req.send();
-    });
-}
-
-

src/dns/caas.js

@@ -0,0 +1,136 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    update: update,
+    getChangeStatus: getChangeStatus,
+    get: get
+};
+
+var assert = require('assert'),
+    config = require('../config.js'),
+    debug = require('debug')('box:dns/caas'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    superagent = require('superagent'),
+    util = require('util'),
+    _ = require('underscore');
+
+function add(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    var fqdn = subdomain !== '' && type === 'TXT' ? subdomain + '.' + config.fqdn() : config.appFqdn(subdomain);
+
+    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .post(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(error);
+            if (result.statusCode === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+            return callback(null, result.body.changeId);
+        });
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var fqdn = subdomain !== '' && type === 'TXT' ? subdomain + '.' + config.fqdn() : config.appFqdn(subdomain);
+
+    debug('get: zoneName: %s subdomain: %s type: %s fqdn: %s', zoneName, subdomain, type, fqdn);
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/domains/' + fqdn)
+        .query({ token: dnsConfig.token, type: type })
+        .end(function (error, result) {
+            if (error && !error.response) return callback(error);
+            if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+            return callback(null, result.body.values);
+        });
+}
+
+function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        if (_.isEqual(values, result)) return callback();
+
+        add(dnsConfig, zoneName, subdomain, type, values, callback);
+    });
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .del(config.apiServerOrigin() + '/api/v1/domains/' + config.appFqdn(subdomain))
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(error);
+            if (result.statusCode === 420) return callback(new SubdomainError(SubdomainError.STILL_BUSY));
+            if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND));
+            if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+            return callback(null);
+        });
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    superagent
+        .get(config.apiServerOrigin() + '/api/v1/domains/' + config.fqdn() + '/status/' + changeId)
+        .query({ token: dnsConfig.token })
+        .end(function (error, result) {
+            if (error && !error.response) return callback(error);
+            if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
+
+            return callback(null, result.body.status);
+        });
+
+}

src/dns/route53.js

@@ -0,0 +1,216 @@
+/* jslint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    get: get,
+    del: del,
+    update: update,
+    getChangeStatus: getChangeStatus
+};
+
+var assert = require('assert'),
+    AWS = require('aws-sdk'),
+    config = require('../config.js'),
+    debug = require('debug')('box:dns/route53'),
+    SubdomainError = require('../subdomains.js').SubdomainError,
+    util = require('util'),
+    _ = require('underscore');
+
+function getDnsCredentials(dnsConfig) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    var credentials = {
+        accessKeyId: dnsConfig.accessKeyId,
+        secretAccessKey: dnsConfig.secretAccessKey,
+        region: dnsConfig.region
+    };
+
+    if (dnsConfig.endpoint) credentials.endpoint = new AWS.Endpoint(dnsConfig.endpoint);
+
+    return credentials;
+}
+
+function getZoneByName(dnsConfig, zoneName, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    route53.listHostedZones({}, function (error, result) {
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+        if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+
+        var zone = result.HostedZones.filter(function (zone) {
+            return zone.Name.slice(0, -1) === zoneName;     // aws zone name contains a '.' at the end
+        })[0];
+
+        if (!zone) return callback(new SubdomainError(SubdomainError.NOT_FOUND, 'no such zone'));
+
+        callback(null, zone);
+    });
+}
+
+function add(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('add: %s for zone %s of type %s with values %j', subdomain, zoneName, type, values);
+
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var records = values.map(function (v) { return { Value: v }; });
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'UPSERT',
+                    ResourceRecordSet: {
+                        Type: type,
+                        Name: fqdn,
+                        ResourceRecords: records,
+                        TTL: 1
+                    }
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error && error.code === 'PriorRequestNotComplete') return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+
+            callback(null, result.ChangeInfo.Id);
+        });
+    });
+}
+
+function update(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    get(dnsConfig, zoneName, subdomain, type, function (error, result) {
+        if (error) return callback(error);
+
+        if (_.isEqual(values, result)) return callback();
+
+        add(dnsConfig, zoneName, subdomain, type, values, callback);
+    });
+}
+
+function get(dnsConfig, zoneName, subdomain, type, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var params = {
+            HostedZoneId: zone.Id,
+            MaxItems: '1',
+            StartRecordName: (subdomain ? subdomain + '.' : '') + zoneName + '.',
+            StartRecordType: type
+        };
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.listResourceRecordSets(params, function (error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+            if (result.ResourceRecordSets.length === 0) return callback(null, [ ]);
+            if (result.ResourceRecordSets[0].Name !== params.StartRecordName && result.ResourceRecordSets[0].Type !== params.StartRecordType) return callback(null, [ ]);
+
+            var values = result.ResourceRecordSets[0].ResourceRecords.map(function (record) { return record.Value; });
+
+            callback(null, values);
+        });
+    });
+}
+
+function del(dnsConfig, zoneName, subdomain, type, values, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof zoneName, 'string');
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    getZoneByName(dnsConfig, zoneName, function (error, zone) {
+        if (error) return callback(error);
+
+        var fqdn = config.appFqdn(subdomain);
+        var records = values.map(function (v) { return { Value: v }; });
+
+        var resourceRecordSet = {
+            Name: fqdn,
+            Type: type,
+            ResourceRecords: records,
+            TTL: 1
+        };
+
+        var params = {
+            ChangeBatch: {
+                Changes: [{
+                    Action: 'DELETE',
+                    ResourceRecordSet: resourceRecordSet
+                }]
+            },
+            HostedZoneId: zone.Id
+        };
+
+        var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+        route53.changeResourceRecordSets(params, function(error, result) {
+            if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+            if (error && error.message && error.message.indexOf('it was not found') !== -1) {
+                debug('del: resource record set not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
+            } else if (error && error.code === 'NoSuchHostedZone') {
+                debug('del: hosted zone not found.', error);
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
+            } else if (error && error.code === 'PriorRequestNotComplete') {
+                debug('del: resource is still busy', error);
+                return callback(new SubdomainError(SubdomainError.STILL_BUSY, error.message));
+            } else if (error && error.code === 'InvalidChangeBatch') {
+                debug('del: invalid change batch. No such record to be deleted.');
+                return callback(new SubdomainError(SubdomainError.NOT_FOUND, error.message));
+            } else if (error) {
+                debug('del: error', error);
+                return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, error.message));
+            }
+
+            callback(null);
+        });
+    });
+}
+
+function getChangeStatus(dnsConfig, changeId, callback) {
+    assert.strictEqual(typeof dnsConfig, 'object');
+    assert.strictEqual(typeof changeId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (changeId === '') return callback(null, 'INSYNC');
+
+    var route53 = new AWS.Route53(getDnsCredentials(dnsConfig));
+    route53.getChange({ Id: changeId }, function (error, result) {
+        if (error && error.code === 'AccessDenied') return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, error.message));
+        if (error) return callback(error);
+
+        callback(null, result.ChangeInfo.Status);
+    });
+}
+

src/docker.js

@@ -1,42 +1,391 @@
 'use strict';
 
-var Docker = require('dockerode'),
+var addons = require('./addons.js'),
-    fs = require('fs'),
+    async = require('async'),
-    os = require('os'),
+    assert = require('assert'),
-    path = require('path'),
+    config = require('./config.js'),
-    url = require('url');
+    constants = require('./constants.js'),
+    debug = require('debug')('box:src/docker.js'),
+    Docker = require('dockerode'),
+    safe = require('safetydance'),
+    semver = require('semver'),
+    util = require('util'),
+    _ = require('underscore');
 
-exports = module.exports = (function () {
+exports = module.exports = {
+    connection: connectionInstance(),
+    downloadImage: downloadImage,
+    createContainer: createContainer,
+    startContainer: startContainer,
+    stopContainer: stopContainer,
+    stopContainerByName: stopContainer,
+    stopContainers: stopContainers,
+    deleteContainer: deleteContainer,
+    deleteContainerByName: deleteContainer,
+    deleteImage: deleteImage,
+    deleteContainers: deleteContainers,
+    createSubcontainer: createSubcontainer,
+    getContainerIdByIp: getContainerIdByIp
+};
+
+function connectionInstance() {
     var docker;
-    var options = connectOptions(); // the real docker
 
     if (process.env.BOX_ENV === 'test') {
         // test code runs a docker proxy on this port
         docker = new Docker({ host: 'http://localhost', port: 5687 });
-    } else {
-        docker = new Docker(options);
-    }
 
         // proxy code uses this to route to the real docker
-    docker.options = options;
+        docker.options = { socketPath: '/var/run/docker.sock' };
+    } else {
+        docker = new Docker({ socketPath: '/var/run/docker.sock' });
+    }
 
     return docker;
-})();
-
-function connectOptions() {
-    if (os.platform() === 'linux') return { socketPath: '/var/run/docker.sock' };
-
-    // boot2docker configuration
-    var DOCKER_CERT_PATH = process.env.DOCKER_CERT_PATH || path.join(process.env.HOME, '.boot2docker/certs/boot2docker-vm');
-    var DOCKER_HOST = process.env.DOCKER_HOST || 'tcp://192.168.59.103:2376';
-
-    return {
-        protocol: 'https',
-        host: url.parse(DOCKER_HOST).hostname,
-        port: url.parse(DOCKER_HOST).port,
-        ca: fs.readFileSync(path.join(DOCKER_CERT_PATH, 'ca.pem')),
-        cert: fs.readFileSync(path.join(DOCKER_CERT_PATH, 'cert.pem')),
-        key: fs.readFileSync(path.join(DOCKER_CERT_PATH, 'key.pem'))
-    };
 }
 
+function debugApp(app, args) {
+    assert(!app || typeof app === 'object');
+
+    var prefix = app ? (app.location || '(bare)') : '(no app)';
+    debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
+function targetBoxVersion(manifest) {
+    if ('targetBoxVersion' in manifest) return manifest.targetBoxVersion;
+
+    if ('minBoxVersion' in manifest) return manifest.minBoxVersion;
+
+    return '99999.99999.99999'; // compatible with the latest version
+}
+
+function pullImage(manifest, callback) {
+    var docker = exports.connection;
+
+    docker.pull(manifest.dockerImage, function (err, stream) {
+        if (err) return callback(new Error('Error connecting to docker. statusCode: %s' + err.statusCode));
+
+        // https://github.com/dotcloud/docker/issues/1074 says each status message
+        // is emitted as a chunk
+        stream.on('data', function (chunk) {
+            var data = safe.JSON.parse(chunk) || { };
+            debug('pullImage %s: %j', manifest.id, data);
+
+            // The information here is useless because this is per layer as opposed to per image
+            if (data.status) {
+            } else if (data.error) {
+                debug('pullImage error %s: %s', manifest.id, data.errorDetail.message);
+            }
+        });
+
+        stream.on('end', function () {
+            debug('downloaded image %s of %s successfully', manifest.dockerImage, manifest.id);
+
+            var image = docker.getImage(manifest.dockerImage);
+
+            image.inspect(function (err, data) {
+                if (err) return callback(new Error('Error inspecting image:' + err.message));
+                if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
+                if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
+
+                debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
+
+                callback(null);
+            });
+        });
+
+        stream.on('error', function (error) {
+            debug('error pulling image %s of %s: %j', manifest.dockerImage, manifest.id, error);
+
+            callback(error);
+        });
+    });
+}
+
+function downloadImage(manifest, callback) {
+    assert.strictEqual(typeof manifest, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('downloadImage %s %s', manifest.id, manifest.dockerImage);
+
+    var attempt = 1;
+
+    async.retry({ times: 10, interval: 15000 }, function (retryCallback) {
+        debug('Downloading image %s %s. attempt: %s', manifest.id, manifest.dockerImage, attempt++);
+
+        pullImage(manifest, function (error) {
+            if (error) console.error(error);
+
+            retryCallback(error);
+        });
+    }, callback);
+}
+
+function createSubcontainer(app, name, cmd, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof name, 'string');
+    assert(!cmd || util.isArray(cmd));
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection,
+        isAppContainer = !cmd;
+
+    var manifest = app.manifest;
+    var developmentMode = !!manifest.developmentMode;
+    var exposedPorts = {}, dockerPortBindings = { };
+    var stdEnv = [
+        'CLOUDRON=1',
+        'WEBADMIN_ORIGIN=' + config.adminOrigin(),
+        'API_ORIGIN=' + config.adminOrigin(),
+        'APP_ORIGIN=https://' + config.appFqdn(app.location),
+        'APP_DOMAIN=' + config.appFqdn(app.location)
+    ];
+
+    // docker portBindings requires ports to be exposed
+    exposedPorts[manifest.httpPort + '/tcp'] = {};
+
+    dockerPortBindings[manifest.httpPort + '/tcp'] = [ { HostIp: '127.0.0.1', HostPort: app.httpPort + '' } ];
+
+    var portEnv = [];
+    for (var e in app.portBindings) {
+        var hostPort = app.portBindings[e];
+        var containerPort = manifest.tcpPorts[e].containerPort || hostPort;
+
+        exposedPorts[containerPort + '/tcp'] = {};
+        portEnv.push(e + '=' + hostPort);
+
+        dockerPortBindings[containerPort + '/tcp'] = [ { HostIp: '0.0.0.0', HostPort: hostPort + '' } ];
+    }
+
+    // first check db record, then manifest
+    var memoryLimit = app.memoryLimit || manifest.memoryLimit;
+
+    // ensure we never go below minimum
+    memoryLimit = memoryLimit < constants.DEFAULT_MEMORY_LIMIT ? constants.DEFAULT_MEMORY_LIMIT : memoryLimit; // 256mb by default
+
+    // developerMode does not restrict memory usage
+    memoryLimit = developmentMode ? 0 : memoryLimit;
+
+    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
+    // this means cloudron exec does not work
+    var isolatedNetworkNs = true;
+
+    addons.getEnvironment(app, function (error, addonEnv) {
+        if (error) return callback(new Error('Error getting addon environment : ' + error));
+
+        var containerOptions = {
+            name: name, // used for filtering logs
+            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
+            // for subcontainers, this should not be set because we already share the network namespace with app container
+            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
+            Tty: isAppContainer,
+            Image: app.manifest.dockerImage,
+            Cmd: (isAppContainer && developmentMode) ? [ '/bin/bash', '-c', 'echo "Development mode. Use cloudron exec to debug. Sleeping" && sleep infinity' ] : cmd,
+            Env: stdEnv.concat(addonEnv).concat(portEnv),
+            ExposedPorts: isAppContainer ? exposedPorts : { },
+            Volumes: { // see also ReadonlyRootfs
+                '/tmp': {},
+                '/run': {}
+            },
+            Labels: {
+                "location": app.location,
+                "appId": app.id,
+                "isSubcontainer": String(!isAppContainer)
+            },
+            HostConfig: {
+                Binds: addons.getBindsSync(app, app.manifest.addons),
+                Memory: memoryLimit / 2,
+                MemorySwap: memoryLimit, // Memory + Swap
+                PortBindings: isAppContainer ? dockerPortBindings : { },
+                PublishAllPorts: false,
+                ReadonlyRootfs: !developmentMode, // see also Volumes in startContainer
+                RestartPolicy: {
+                    "Name": isAppContainer ? "always" : "no",
+                    "MaximumRetryCount": 0
+                },
+                CpuShares: 512, // relative to 1024 for system processes
+                VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
+                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
+                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
+                SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
+            }
+        };
+        containerOptions = _.extend(containerOptions, options);
+
+        debugApp(app, 'Creating container for %s with options %j', app.manifest.dockerImage, containerOptions);
+
+        docker.createContainer(containerOptions, callback);
+    });
+}
+
+function createContainer(app, callback) {
+    createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback);
+}
+
+function startContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    var container = docker.getContainer(containerId);
+    debug('Starting container %s', containerId);
+
+    container.start(function (error) {
+        if (error && error.statusCode !== 304) return callback(new Error('Error starting container :' + error));
+
+        return callback(null);
+    });
+}
+
+function stopContainer(containerId, callback) {
+    assert(!containerId || typeof containerId === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!containerId) {
+        debug('No previous container to stop');
+        return callback();
+    }
+
+    var docker = exports.connection;
+    var container = docker.getContainer(containerId);
+    debug('Stopping container %s', containerId);
+
+    var options = {
+        t: 10 // wait for 10 seconds before killing it
+    };
+
+    container.stop(options, function (error) {
+        if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error stopping container:' + error));
+
+        debug('Waiting for container ' + containerId);
+
+        container.wait(function (error, data) {
+            if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new Error('Error waiting on container:' + error));
+
+            debug('Container %s stopped with status code [%s]', containerId, data ? String(data.StatusCode) : '');
+
+            return callback(null);
+        });
+    });
+}
+
+function deleteContainer(containerId, callback) {
+    assert(!containerId || typeof containerId === 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('deleting container %s', containerId);
+
+    if (containerId === null) return callback(null);
+
+    var docker = exports.connection;
+    var container = docker.getContainer(containerId);
+
+    var removeOptions = {
+        force: true, // kill container if it's running
+        v: true // removes volumes associated with the container (but not host mounts)
+    };
+
+    container.remove(removeOptions, function (error) {
+        if (error && error.statusCode === 404) return callback(null);
+
+        if (error) debug('Error removing container %s : %j', containerId, error);
+
+        callback(error);
+    });
+}
+
+function deleteContainers(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    debug('deleting containers of %s', appId);
+
+    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            deleteContainer(container.Id, iteratorDone);
+        }, callback);
+    });
+}
+
+function stopContainers(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var docker = exports.connection;
+
+    debug('stopping containers of %s', appId);
+
+    docker.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            stopContainer(container.Id, iteratorDone);
+        }, callback);
+    });
+}
+
+function deleteImage(manifest, callback) {
+    assert(!manifest || typeof manifest === 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var dockerImage = manifest ? manifest.dockerImage : null;
+    if (!dockerImage) return callback(null);
+
+    var docker = exports.connection;
+
+    var removeOptions = {
+        force: false, // might be shared with another instance of this app
+        noprune: false // delete untagged parents
+    };
+
+    // registry v1 used to pull down all *tags*. this meant that deleting image by tag was not enough (since that
+    // just removes the tag). we used to remove the image by id. this is not required anymore because aliases are
+    // not created anymore after https://github.com/docker/docker/pull/10571
+    docker.getImage(dockerImage).remove(removeOptions, function (error) {
+        if (error && error.statusCode === 404) return callback(null);
+        if (error && error.statusCode === 409) return callback(null); // another container using the image
+
+        if (error) debug('Error removing image %s : %j', dockerImage, error);
+
+        callback(error);
+    });
+}
+
+function getContainerIdByIp(ip, callback) {
+    assert.strictEqual(typeof ip, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('get container by ip %s', ip);
+
+    var docker = exports.connection;
+
+    docker.listNetworks({}, function (error, result) {
+        if (error) return callback(error);
+
+        var bridge;
+        result.forEach(function (n) {
+            if (n.Name === 'bridge') bridge = n;
+        });
+
+        if (!bridge) return callback(new Error('Unable to find the bridge network'));
+
+        var containerId;
+        for (var id in bridge.Containers) {
+            if (bridge.Containers[id].IPv4Address.indexOf(ip) === 0) {
+                containerId = id;
+                break;
+            }
+        }
+        if (!containerId) return callback(new Error('No container with that ip'));
+
+        debug('found container %s with ip %s', containerId, ip);
+
+        callback(null, containerId);
+    });
+}

src/groupdb.js

@@ -0,0 +1,202 @@
+'use strict';
+
+exports = module.exports = {
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+    add: add,
+    del: del,
+    count: count,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror');
+
+var GROUPS_FIELDS = [ 'id', 'name' ].join(',');
+
+function get(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups WHERE id = ?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null, result[0]);
+    });
+}
+
+function getWithMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' WHERE groups.id = ? ' +
+                    ' GROUP BY groups.id', [ groupId ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        var result = results[0];
+        result.userIds = result.userIds ? result.userIds.split(',') : [ ];
+
+        callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function add(id, name, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var data = [ id, name ];
+    database.query('INSERT INTO groups (id, name) VALUES (?, ?)',
+           data, function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function del(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM groupMembers WHERE groupId = ?', args: [ id ] });
+    queries.push({ query: 'DELETE FROM groups WHERE id = ?', args: [ id ] });
+
+    database.transaction(queries, function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result[1].affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(error);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT COUNT(*) AS total FROM groups', function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        return callback(null, result[0].total);
+    });
+}
+
+function clear(callback) {
+    database.query('DELETE FROM groupMembers', function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        database.query('DELETE FROM groups WHERE id != ?', [ 'admin' ], function (error) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            callback(error);
+        });
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT userId FROM groupMembers WHERE groupId=?', [ groupId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.userId; }));
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT groupId FROM groupMembers WHERE userId=? ORDER BY groupId', [ userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        // if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND)); // need to differentiate group with no members and invalid groupId
+
+        callback(error, result.map(function (r) { return r.groupId; }));
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    var queries = [ ];
+    queries.push({ query: 'DELETE from groupMembers WHERE userId = ?', args: [ userId ] });
+    groupIds.forEach(function (gid) {
+        queries.push({ query: 'INSERT INTO groupMembers (groupId, userId) VALUES (? , ?)', args: [ gid, userId ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ groupId, userId ], function (error, result) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM groupMembers WHERE groupId = ? AND userId = ?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT 1 FROM groupMembers WHERE groupId=? AND userId=?', [ groupId, userId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, result.length !== 0);
+    });
+}

src/groups.js

@@ -0,0 +1,210 @@
+/* jshint node:true */
+
+'use strict';
+
+exports = module.exports = {
+    GroupError: GroupError,
+
+    create: create,
+    remove: remove,
+    get: get,
+    getWithMembers: getWithMembers,
+    getAll: getAll,
+
+    getMembers: getMembers,
+    addMember: addMember,
+    removeMember: removeMember,
+    isMember: isMember,
+
+    getGroups: getGroups,
+    setGroups: setGroups,
+
+    ADMIN_GROUP_ID: 'admin' // see db migration code and groupdb._clear
+};
+
+var assert = require('assert'),
+    DatabaseError = require('./databaseerror.js'),
+    groupdb = require('./groupdb.js'),
+    util = require('util');
+
+// http://dustinsenos.com/articles/customErrorsInNode
+// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
+function GroupError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(GroupError, Error);
+GroupError.INTERNAL_ERROR = 'Internal Error';
+GroupError.ALREADY_EXISTS = 'Already Exists';
+GroupError.NOT_FOUND = 'Not Found';
+GroupError.BAD_NAME = 'Bad name';
+GroupError.NOT_EMPTY = 'Not Empty';
+GroupError.NOT_ALLOWED = 'Not Allowed';
+
+function validateGroupname(name) {
+    assert.strictEqual(typeof name, 'string');
+    var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
+
+    if (name.length <= 2) return new GroupError(GroupError.BAD_NAME, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_NAME, 'name too long');
+
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_NAME, 'name can only have A-Za-z0-9_-');
+
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_NAME, 'name is reserved');
+
+    return null;
+}
+
+function create(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var error = validateGroupname(name);
+    if (error) return callback(error);
+
+    groupdb.add(name /* id */, name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new GroupError(GroupError.ALREADY_EXISTS));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null, { id: name, name: name });
+    });
+}
+
+function remove(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // never allow admin group to be deleted
+    if (id === exports.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
+
+    groupdb.del(id, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function get(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getWithMembers(id, callback) {
+    assert.strictEqual(typeof id, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getWithMembers(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAll(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getMembers(groupId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getMembers(groupId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getGroups(userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
+function setGroups(userId, groupIds, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert(Array.isArray(groupIds));
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.setGroups(userId, groupIds, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function addMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.addMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function removeMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.removeMember(groupId, userId, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null);
+    });
+}
+
+function isMember(groupId, userId, callback) {
+    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.isMember(groupId, userId, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}

src/janitor.js

@@ -0,0 +1,103 @@
+'use strict';
+
+var assert = require('assert'),
+    async = require('async'),
+    authcodedb = require('./authcodedb.js'),
+    debug = require('debug')('box:src/janitor'),
+    docker = require('./docker.js').connection,
+    tokendb = require('./tokendb.js');
+
+exports = module.exports = {
+    cleanupTokens: cleanupTokens,
+    cleanupDockerVolumes: cleanupDockerVolumes
+};
+
+var NOOP_CALLBACK = function () { };
+
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
+
+            callback();
+        });
+    };
+}
+
+function cleanupExpiredTokens(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    tokendb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired tokens.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupExpiredAuthCodes(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    authcodedb.delExpired(function (error, result) {
+        if (error) return callback(error);
+
+        debug('Cleaned up %s expired authcodes.', result);
+
+        callback(null);
+    });
+}
+
+function cleanupTokens(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    debug('Cleaning up expired tokens');
+
+    async.series([
+        ignoreError(cleanupExpiredTokens),
+        ignoreError(cleanupExpiredAuthCodes)
+    ], callback);
+}
+
+function cleanupTmpVolume(containerInfo, callback) {
+    assert.strictEqual(typeof containerInfo, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var cmd = 'find /tmp -mtime +10 -exec rm -rf {} +'.split(' '); // 10 days old
+
+    debug('cleanupTmpVolume %j', containerInfo.Names);
+
+    docker.getContainer(containerInfo.Id).exec({ Cmd: cmd, AttachStdout: true, AttachStderr: true, Tty: false }, function (error, execContainer) {
+        if (error) return callback(new Error('Failed to exec container : ' + error.message));
+
+        execContainer.start(function(err, stream) {
+            if (error) return callback(new Error('Failed to start exec container : ' + error.message));
+
+            stream.on('error', callback);
+            stream.on('end', callback);
+
+            stream.setEncoding('utf8');
+            stream.pipe(process.stdout);
+        });
+    });
+}
+
+function cleanupDockerVolumes(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    callback = callback || NOOP_CALLBACK;
+
+    debug('Cleaning up docker volumes');
+
+    docker.listContainers({ all: 0 }, function (error, containers) {
+        if (error) return callback(error);
+
+        async.eachSeries(containers, function (container, iteratorDone) {
+            cleanupTmpVolume(container, function (error) {
+                if (error) debug('Error cleaning tmp: %s', error);
+
+                iteratorDone(); // intentionally ignore error
+            });
+        }, callback);
+    });
+}

src/ldap.js

@@ -1,10 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    start: start
+    start: start,
+    stop: stop
 };
 
 var assert = require('assert'),
+    apps = require('./apps.js'),
     config = require('./config.js'),
     debug = require('debug')('box:ldap'),
     user = require('./user.js'),
@@ -25,17 +27,27 @@ var gLogger = {
 };
 
 var GROUP_USERS_DN = 'cn=users,ou=groups,dc=cloudron';
-var GROUP_ADMINS_DN = 'cn=admin,ou=groups,dc=cloudron';
+var GROUP_ADMINS_DN = 'cn=admins,ou=groups,dc=cloudron';
+
+function getAppByRequest(req, callback) {
+    var sourceIp = req.connection.ldap.id.split(':')[0];
+    if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
+
+    apps.getByIpAddress(sourceIp, function (error, app) {
+        // we currently allow access in case we can't find the source app
+        callback(null, app || null);
+    });
+}
 
 function start(callback) {
-    assert(typeof callback === 'function');
+    assert.strictEqual(typeof callback, 'function');
 
     gServer = ldap.createServer({ log: gLogger });
 
     gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('ldap user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+        debug('user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
 
-        user.list(function (error, result){
+        user.list(function (error, result) {
             if (error) return next(new ldap.OperationsError(error.toString()));
 
             // send user objects
@@ -53,7 +65,7 @@ function start(callback) {
                         cn: entry.id,
                         uid: entry.id,
                         mail: entry.email,
-                        displayname: entry.username,
+                        displayname: entry.displayName || entry.username,
                         username: entry.username,
                         samaccountname: entry.username,      // to support ActiveDirectory clients
                         memberof: groups
@@ -62,7 +74,6 @@ function start(callback) {
 
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
                     res.send(tmp);
-                    debug('ldap user send:', tmp);
                 }
             });
 
@@ -71,7 +82,7 @@ function start(callback) {
     });
 
     gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('ldap group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+        debug('group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
 
         user.list(function (error, result){
             if (error) return next(new ldap.OperationsError(error.toString()));
@@ -99,7 +110,6 @@ function start(callback) {
 
                 if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && req.filter.matches(tmp.attributes)) {
                     res.send(tmp);
-                    debug('ldap group send:', tmp);
                 }
             });
 
@@ -107,19 +117,56 @@ function start(callback) {
         });
     });
 
-    gServer.bind('dc=cloudron', function(req, res, next) {
+    gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
-        debug('ldap bind: %s', req.dn.toString());
+        // TODO: validate password
+        debug('application bind: %s', req.dn.toString());
+        res.end();
+    });
 
-        if (!req.dn.rdns[0].cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
+        debug('user bind: %s', req.dn.toString());
 
-        user.verify(req.dn.rdns[0].cn, req.credentials || '', function (error, result) {
+        // extract the common name which might have different attribute names
+        var attributeName = Object.keys(req.dn.rdns[0])[0];
+        var commonName = req.dn.rdns[0][attributeName];
+        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+        // if mail is specified, enforce mail check, otherwise allow both
+        var api = (commonName.indexOf('@') === -1) && (attributeName !== 'mail')  ? user.verify : user.verifyWithEmail;
+
+        // TODO this should be done after we verified the app has access to avoid leakage of user existence
+        api(commonName, req.credentials || '', function (error, userObject) {
             if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error));
 
+            getAppByRequest(req, function (error, app) {
+                if (error) return next(error);
+
+                if (!app) {
+                    debug('no app found for this container, allow access');
+                    return res.end();
+                }
+
+                apps.hasAccessTo(app, userObject, function (error, result) {
+                    if (error) return next(new ldap.OperationsError(error.toString()));
+
+                    // we return no such object, to avoid leakage of a users existence
+                    if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
                     res.end();
                 });
             });
+        });
+    });
 
     gServer.listen(config.get('ldapPort'), callback);
 }
+
+function stop(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    gServer.close();
+
+    callback();
+}

src/locker.js

@@ -9,6 +9,7 @@ function Locker() {
     this._operation = null;
     this._timestamp = null;
     this._watcherId = -1;
+    this._lockDepth = 0; // recursive locks
 }
 util.inherits(Locker, EventEmitter);
 
@@ -24,6 +25,7 @@ Locker.prototype.lock = function (operation) {
     if (this._operation !== null) return new Error('Already locked for ' + this._operation);
 
     this._operation = operation;
+    ++this._lockDepth;
     this._timestamp = new Date();
     var that = this;
     this._watcherId = setInterval(function () { debug('Lock unreleased %s', that._operation); }, 1000 * 60 * 5);
@@ -35,17 +37,31 @@ Locker.prototype.lock = function (operation) {
     return null;
 };
 
+Locker.prototype.recursiveLock = function (operation) {
+    if (this._operation === operation) {
+        ++this._lockDepth;
+        debug('Re-acquired : %s Depth : %s', this._operation, this._lockDepth);
+        return null;
+    }
+
+    return this.lock(operation);
+};
+
 Locker.prototype.unlock = function (operation) {
     assert.strictEqual(typeof operation, 'string');
 
     if (this._operation !== operation) throw new Error('Mismatched unlock. Current lock is for ' + this._operation); // throw because this is a programming error
 
+    if (--this._lockDepth === 0) {
         debug('Released : %s', this._operation);
 
         this._operation = null;
         this._timestamp = null;
         clearInterval(this._watcherId);
         this._watcherId = -1;
+    } else {
+        debug('Recursive lock released : %s. Depth : %s', this._operation, this._lockDepth);
+    }
 
     this.emit('unlocked', operation);
 

src/mail_templates/app_down.ejs

@@ -5,8 +5,7 @@ Dear Admin,
 The application titled '<%= title %>' that you installed at <%= appFqdn %>
 is not responding.
 
-This is most likely a problem in the application. Please report this issue to
+This is most likely a problem in the application.
-support@cloudron.io (by forwarding this email).
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mail_templates/app_update_available.ejs

@@ -2,12 +2,15 @@
 
 Dear Admin,
 
-A new version of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+A new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
 
-Please update at your convenience at <%= webadminUrl %>.
+The app will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
+
+Changes:
+<%= updateInfo.manifest.changelog %>
 
 Thank you,
-Update Manager
+your Cloudron
 
 <% } else { %>
 

src/mail_templates/box_update_available.ejs

@@ -2,9 +2,9 @@
 
 Dear Admin,
 
-A new version of Cloudron <%= fqdn %> is available!
+Version <%= newBoxVersion %> of Cloudron <%= fqdn %> is now available!
 
-Please update at your convenience at <%= webadminUrl %>.
+Your Cloudron will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
 
 Changelog:
 <% for (var i = 0; i < changelog.length; i++) { %>
@@ -12,7 +12,7 @@ Changelog:
 <% } %>
 
 Thank you,
-Update Manager
+your Cloudron
 
 <% } else { %>
 

src/mail_templates/certificate_renewed.ejs

@@ -0,0 +1,16 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+<% if (message) { %>
+<%= domain %> was not renewed.
+
+<%- message %>
+<% } else { %>
+<%= domain %> was renewed.
+<% } %>
+Thank you,
+Your Cloudron
+<% } else { %>
+
+<% } %>
+

src/mail_templates/crash_notification.ejs

@@ -2,7 +2,7 @@
 
 Dear Cloudron Team,
 
-unfortunately the <%= program %> on <%= fqdn %> crashed unexpectedly!
+Unfortunately <%= program %> on <%= fqdn %> crashed unexpectedly!
 
 Please see some excerpt of the logs below.
 
@@ -11,7 +11,7 @@ Your Cloudron
 
 -------------------------------------
 
-<%= context %>
+<%- context %>
 
 <% } else { %>
 

src/mail_templates/out_of_disk_space.ejs

@@ -0,0 +1,19 @@
+<%if (format === 'text') { %>
+
+Dear Cloudron Team,
+
+<%= fqdn %> is running out of disk space.
+
+Please see some excerpts of the logs below.
+
+Thank you,
+Your Cloudron
+
+-------------------------------------
+
+<%- message %>
+
+<% } else { %>
+
+<% } %>
+

src/mail_templates/user_added.ejs

@@ -0,0 +1,23 @@
+<%if (format === 'text') { %>
+
+Dear Admin,
+
+User with name '<%= username %>' (<%= email %>) was added in the Cloudron at <%= fqdn %>.
+
+You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
+
+<% if (inviteLink) { %>
+As requested, this user has not been sent an invitation email.
+
+To set a password and perform any configuration on behalf of the user, please use this link:
+<%= inviteLink %>
+
+<% } %>
+
+Thank you,
+User Manager
+
+<% } else { %>
+
+<% } %>
+