app.portBindings and newManifest.tcpPorts may be null

2015-07-20 00:09:47 -07:00
commit df9d321ac3
243 changed files with 42623 additions and 0 deletions
--- a/setup/DESIGN.md
+++ b/setup/DESIGN.md
@@ -0,0 +1,57 @@
+This document gives the design of this setup code.
+
+box code should be delivered in the form of a (docker) container.
+This is not the case currently but we want to do structure the code
+in spirit that way.
+
+### container.sh
+This contains code that essential goes into Dockerfile.
+
+This file contains static configuration over a base image. Currently,
+the yellowtent user is created in the installer base image but it
+could very well be placed here.
+
+The idea is that the installer would simply remove the old box container
+and replace it with a new one for an update.
+
+Because we do not package things as Docker yet, we should be careful
+about the code here. We have to expect remains of an older setup code.
+For example, older supervisor or nginx configs might be around.
+
+The config directory is _part_ of the container and is not a VOLUME.
+Which is to say that the files will be nuked from one update to the next.
+
+The data directory is a VOLUME. Contents of this directory are expected
+to survive an update. This is a good place to place config files that
+are "dynamic" and need to survive restarts. For example, the infra
+version (see below) or the mysql/postgresql data etc.
+
+### start.sh
+  * It is called in 3 modes - new, update, restore.
+
+  * The first thing this does is to do the static container.sh setup.
+
+  * It then downloads any box restore data and restores the box db from the
+    backup.
+
+  * It then proceeds to call the db-migrate script.
+
+  * It then does dynamic configuration like setting up nginx, collectd.
+
+  * It then setups up the cloud infra (setup_infra.sh) and creates cloudron.conf.
+
+  * supervisor is then started
+
+setup_infra.sh
+This setups containers like graphite, mail and the addons containers.
+
+Containers are relaunched based on the INFRA_VERSION. The script compares
+the version here with the version in the file DATA_DIR/INFRA_VERSION.
+
+If they match, the containers are not recreated and nothing is to be done.
+nginx, collectd configs are part of data already and containers are running.
+
+If they do not match, it deletes all containers (including app containers) and starts
+them all afresh. Important thing here is that, DATA_DIR is never removed across
+updates. So, it is only the containers being recreated and not the data.
+
--- a/setup/INFRA_VERSION
+++ b/setup/INFRA_VERSION
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# If you change the infra version, be sure to put a warning
+# in the change log
+
+INFRA_VERSION=4
--- a/setup/argparser.sh
+++ b/setup/argparser.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+json="${script_dir}/../node_modules/.bin/json"
+
+arg_restore_url=""
+arg_restore_key=""
+arg_box_versions_url=""
+arg_tls_cert=""
+arg_tls_key=""
+arg_api_server_origin=""
+arg_web_server_origin=""
+arg_fqdn=""
+arg_token=""
+arg_version=""
+arg_is_custom_domain="false"
+arg_developer_mode=""
+arg_retire="false"
+arg_model=""
+
+args=$(getopt -o "" -l "data:,retire" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --retire)
+        arg_retire="true"
+        shift
+        ;;
+    --data)
+        # only read mandatory non-empty parameters here
+        read -r arg_api_server_origin arg_web_server_origin arg_fqdn arg_token arg_is_custom_domain arg_box_versions_url arg_version <<EOF
+        $(echo "$2" | $json apiServerOrigin webServerOrigin fqdn token isCustomDomain boxVersionsUrl version | tr '\n' ' ')
+EOF
+        # read possibly empty parameters here
+        arg_tls_cert=$(echo "$2" | $json tlsCert)
+        arg_tls_key=$(echo "$2" | $json tlsKey)
+
+        arg_developer_mode=$(echo "$2" | $json developerMode)
+        [[ "${arg_developer_mode}" == "" ]] && arg_developer_mode="false"
+
+        arg_restore_url=$(echo "$2" | $json restoreUrl)
+        [[ "${arg_restore_url}" == "null" ]] && arg_restore_url=""
+
+        arg_restore_key=$(echo "$2" | $json restoreKey)
+        [[ "${arg_restore_key}" == "null" ]] && arg_restore_key=""
+
+        arg_model=$(echo "$2" | $json model)
+        [[ "${arg_model}" == "null" ]] && arg_model=""
+
+        shift 2
+        ;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+echo "Parsed arguments:"
+echo "restore url: ${arg_restore_url}"
+echo "restore key: ${arg_restore_key}"
+echo "box versions url: ${arg_box_versions_url}"
+echo "api server: ${arg_api_server_origin}"
+echo "web server: ${arg_web_server_origin}"
+echo "fqdn: ${arg_fqdn}"
+echo "token: ${arg_token}"
+echo "version: ${arg_version}"
+echo "custom domain: ${arg_is_custom_domain}"
+echo "tls cert: ${arg_tls_cert}"
+echo "tls key: ${arg_tls_key}"
+echo "developer mode: ${arg_developer_mode}"
+echo "model: ${arg_model}"
--- a/setup/container.sh
+++ b/setup/container.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# This file can be used in Dockerfile
+
+readonly container_files="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/container"
+
+readonly CONFIG_DIR="/home/yellowtent/configs"
+readonly DATA_DIR="/home/yellowtent/data"
+
+########## create config directory
+rm -rf "${CONFIG_DIR}"
+sudo -u yellowtent mkdir "${CONFIG_DIR}"
+
+########## logrotate (default ubuntu runs this daily)
+rm -rf /etc/logrotate.d/*
+cp -r "${container_files}/logrotate/." /etc/logrotate.d/
+
+########## supervisor
+rm -rf /etc/supervisor/*
+cp -r "${container_files}/supervisor/." /etc/supervisor/
+
+########## sudoers
+rm /etc/sudoers.d/*
+cp "${container_files}/sudoers" /etc/sudoers.d/yellowtent
+
+########## collectd
+rm -rf /etc/collectd
+ln -sfF "${DATA_DIR}/collectd" /etc/collectd
+
+########## nginx
+# link nginx config to system config
+unlink /etc/nginx 2>/dev/null || rm -rf /etc/nginx
+ln -s "${DATA_DIR}/nginx" /etc/nginx
+
+########## Enable services
+update-rc.d -f collectd defaults
+
--- a/setup/container/logrotate/cloudron
+++ b/setup/container/logrotate/cloudron
@@ -0,0 +1,6 @@
+/var/log/cloudron/*log {
+    missingok
+    notifempty
+    size 100k
+    nocompress
+}
--- a/setup/container/logrotate/supervisor
+++ b/setup/container/logrotate/supervisor
@@ -0,0 +1,7 @@
+/var/log/supervisor/*log {
+    missingok
+    copytruncate
+    notifempty
+    size 100k
+    nocompress
+}
--- a/setup/container/sudoers
+++ b/setup/container/sudoers
@@ -0,0 +1,26 @@
+Defaults!/home/yellowtent/box/src/scripts/createappdir.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/createappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/rmappdir.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmappdir.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadnginx.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadnginx.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupbox.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupbox.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupapp.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/restoreapp.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restoreapp.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reboot.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reboot.sh
+
+Defaults!/home/yellowtent/box/src/scripts/reloadcollectd.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/reloadcollectd.sh
+
+Defaults!/home/yellowtent/box/src/scripts/backupswap.sh env_keep="HOME NODE_ENV"
+yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/backupswap.sh
--- a/setup/container/supervisor/conf.d/apphealthtask.conf
+++ b/setup/container/supervisor/conf.d/apphealthtask.conf
@@ -0,0 +1,10 @@
+[program:apphealthtask]
+command=/usr/bin/node "/home/yellowtent/box/apphealthtask.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/apphealthtask.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
--- a/setup/container/supervisor/conf.d/box.conf
+++ b/setup/container/supervisor/conf.d/box.conf
@@ -0,0 +1,10 @@
+[program:box]
+command=/usr/bin/node "/home/yellowtent/box/app.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/box.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*,connect-lastmile",NODE_ENV="cloudron"
--- a/setup/container/supervisor/conf.d/crashnotifier.conf
+++ b/setup/container/supervisor/conf.d/crashnotifier.conf
@@ -0,0 +1,11 @@
+[eventlistener:crashnotifier]
+command=/usr/bin/node "/home/yellowtent/box/crashnotifier.js"
+events=PROCESS_STATE
+autostart=true
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/log/supervisor/crashnotifier.log
+stderr_logfile_maxbytes=50MB
+stderr_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",NODE_ENV="cloudron"
--- a/setup/container/supervisor/conf.d/janitor.conf
+++ b/setup/container/supervisor/conf.d/janitor.conf
@@ -0,0 +1,10 @@
+[program:janitor]
+command=/usr/bin/node "/home/yellowtent/box/janitor.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/janitor.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
--- a/setup/container/supervisor/conf.d/oauthproxy.conf
+++ b/setup/container/supervisor/conf.d/oauthproxy.conf
@@ -0,0 +1,10 @@
+[program:oauthproxy]
+command=/usr/bin/node "/home/yellowtent/box/oauthproxy.js"
+autostart=true
+autorestart=true
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/oauthproxy.log
+stdout_logfile_maxbytes=50MB
+stdout_logfile_backups=2
+user=yellowtent
+environment=HOME="/home/yellowtent",USER="yellowtent",DEBUG="box*",NODE_ENV="cloudron"
--- a/setup/container/supervisor/supervisord.conf
+++ b/setup/container/supervisor/supervisord.conf
@@ -0,0 +1,33 @@
+; supervisor config file
+
+; http://coffeeonthekeyboard.com/using-supervisorctl-with-linux-permissions-but-without-root-or-sudo-977/
+[inet_http_server]
+port = 127.0.0.1:9001
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+nodaemon = false
+childlogdir = /var/log/supervisor/
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=http://127.0.0.1:9001
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = conf.d/*.conf
+
--- a/setup/splashpage.sh
+++ b/setup/splashpage.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly BOX_SRC_DIR="/home/yellowtent/box"
+readonly DATA_DIR="/home/yellowtent/data"
+
+source "${script_dir}/INFRA_VERSION" # this injects INFRA_VERSION
+
+echo "Setting up nginx update page"
+
+source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
+
+# copy the website
+rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
+cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
+
+# create nginx config
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+    rm -f ${DATA_DIR}/nginx/applications/*
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"~^(.+)\$\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+else
+    # keep this is sync with config.js appFqdn()
+    readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+    admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+
+    ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+        -O "{ \"vhost\": \"${admin_fqdn}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+fi
+
+echo '{ "update": { "percent": "10", "message": "Updating cloudron software" }, "backup": null }' > "${SETUP_WEBSITE_DIR}/progress.json"
+
+nginx -s reload
--- a/setup/start.sh
+++ b/setup/start.sh
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+echo "==== Cloudron Start ===="
+
+readonly USER="yellowtent"
+readonly BOX_SRC_DIR="/home/${USER}/box"
+readonly DATA_DIR="/home/${USER}/data"
+readonly CONFIG_DIR="/home/${USER}/configs"
+readonly SETUP_PROGRESS_JSON="/home/yellowtent/setup/website/progress.json"
+readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
+
+readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
+
+# keep this is sync with config.js appFqdn()
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+
+readonly is_update=$([[ -d "${DATA_DIR}/box" ]] && echo "true" || echo "false")
+
+set_progress() {
+    local percent="$1"
+    local message="$2"
+
+    echo "==== ${percent} - ${message} ===="
+    (echo "{ \"update\": { \"percent\": \"${percent}\", \"message\": \"${message}\" }, \"backup\": {} }" > "${SETUP_PROGRESS_JSON}") 2> /dev/null || true # as this will fail in non-update mode
+}
+
+set_progress "1" "Create container"
+$script_dir/container.sh
+
+set_progress "10" "Ensuring directories"
+# keep these in sync with paths.js
+[[ "${is_update}" == "false" ]] && btrfs subvolume create "${DATA_DIR}/box"
+mkdir -p "${DATA_DIR}/box/appicons"
+mkdir -p "${DATA_DIR}/box/mail"
+mkdir -p "${DATA_DIR}/graphite"
+
+mkdir -p "${DATA_DIR}/snapshots"
+mkdir -p "${DATA_DIR}/addons"
+mkdir -p "${DATA_DIR}/collectd/collectd.conf.d"
+
+# bookkeep the version as part of data
+echo "{ \"version\": \"${arg_version}\", \"boxVersionsUrl\": \"${arg_box_versions_url}\" }" > "${DATA_DIR}/box/version"
+
+# remove old snapshots. if we do want to keep this around, we will have to fix the chown -R below
+# which currently fails because these are readonly fs
+echo "Cleaning up snapshots"
+find "${DATA_DIR}/snapshots" -mindepth 1 -maxdepth 1 | xargs --no-run-if-empty btrfs subvolume delete
+
+readonly mysql_root_password="password"
+mysqladmin -u root -ppassword password password # reset default root password
+mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
+
+if [[ -n "${arg_restore_url}" ]]; then
+    set_progress "15" "Downloading restore data"
+
+    echo "Downloading backup: ${arg_restore_url} and key: ${arg_restore_key}"
+
+    while true; do
+        if $curl -L "${arg_restore_url}" | openssl aes-256-cbc -d -pass "pass:${arg_restore_key}" | tar -zxf - -C "${DATA_DIR}/box"; then break; fi
+        echo "Failed to download data, trying again"
+    done
+
+    set_progress "21" "Setting up MySQL"
+    if [[ -f "${DATA_DIR}/box/box.mysqldump" ]]; then
+        echo "Importing existing database into MySQL"
+        mysql -u root -p${mysql_root_password} box < "${DATA_DIR}/box/box.mysqldump"
+    fi
+fi
+
+set_progress "25" "Migrating data"
+sudo -u "${USER}" -H bash <<EOF
+set -eu
+cd "${BOX_SRC_DIR}"
+NODE_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@localhost/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
+EOF
+
+set_progress "28" "Setup collectd"
+cp "${script_dir}/start/collectd.conf" "${DATA_DIR}/collectd/collectd.conf"
+service collectd restart
+
+set_progress "30" "Setup nginx"
+# setup naked domain to use admin by default. app restoration will overwrite this config
+mkdir -p "${DATA_DIR}/nginx/applications"
+cp "${script_dir}/start/nginx/mime.types" "${DATA_DIR}/nginx/mime.types"
+
+# generate the main nginx config file
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/nginx.ejs" \
+    -O "{ \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/nginx.conf"
+
+# generate these for update code paths as well to overwrite splash
+${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
+    -O "{ \"vhost\": \"${admin_fqdn}\", \"endpoint\": \"admin\", \"sourceDir\": \"${BOX_SRC_DIR}\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
+
+mkdir -p "${DATA_DIR}/nginx/cert"
+echo "${arg_tls_cert}" > ${DATA_DIR}/nginx/cert/host.cert
+echo "${arg_tls_key}" > ${DATA_DIR}/nginx/cert/host.key
+
+set_progress "33" "Changing ownership"
+chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons"
+
+set_progress "40" "Setting up infra"
+${script_dir}/start/setup_infra.sh "${arg_fqdn}"
+
+set_progress "65" "Creating cloudron.conf"
+admin_origin="https://${admin_fqdn}"
+sudo -u yellowtent -H bash <<EOF
+set -eu
+echo "Creating cloudron.conf"
+cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
+{
+    "version": "${arg_version}",
+    "token": "${arg_token}",
+    "apiServerOrigin": "${arg_api_server_origin}",
+    "webServerOrigin": "${arg_web_server_origin}",
+    "fqdn": "${arg_fqdn}",
+    "isCustomDomain": ${arg_is_custom_domain},
+    "boxVersionsUrl": "${arg_box_versions_url}",
+    "adminEmail": "admin@${arg_fqdn}",
+    "database": {
+        "hostname": "localhost",
+        "username": "root",
+        "password": "${mysql_root_password}",
+        "port": 3306,
+        "name": "box"
+    },
+    "model": "${arg_model}",
+    "developerMode": ${arg_developer_mode}
+}
+CONF_END
+
+echo "Creating config.json for webadmin"
+cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
+{
+    "webServerOrigin": "${arg_web_server_origin}"
+}
+CONF_END
+EOF
+
+# Add webadmin oauth client
+# The domain might have changed, therefor we have to update the record
+# !!! This needs to be in sync with the webadmin, specifically login_callback.js
+echo "Add webadmin oauth cient"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+
+echo "Add localhost test oauth cient"
+ADMIN_SCOPES="root,developer,profile,users,apps,settings,roleUser"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+
+set_progress "80" "Reloading supervisor"
+# looks like restarting supervisor completely is the only way to reload it
+service supervisor stop || true
+
+echo -n "Waiting for supervisord to stop"
+while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
+    echo -n "."
+    sleep 1
+done
+echo ""
+
+echo "Starting supervisor"
+
+service supervisor start
+
+sleep 2 # give supervisor sometime to start the processes
+
+set_progress "85" "Reloading nginx"
+nginx -s reload
+
+set_progress "100" "Done"
+
--- a/setup/start/collectd.conf
+++ b/setup/start/collectd.conf
@@ -0,0 +1,279 @@
+# Config file for collectd(1).
+#
+# Some plugins need additional configuration and are disabled by default.
+# Please read collectd.conf(5) for details.
+#
+# You should also read /usr/share/doc/collectd-core/README.Debian.plugins
+# before enabling any more plugins.
+
+##############################################################################
+# Global                                                                     #
+#----------------------------------------------------------------------------#
+# Global settings for the daemon.                                            #
+##############################################################################
+
+Hostname "localhost"
+#FQDNLookup true
+#BaseDir "/var/lib/collectd"
+#PluginDir "/usr/lib/collectd"
+#TypesDB "/usr/share/collectd/types.db" "/etc/collectd/my_types.db"
+
+#----------------------------------------------------------------------------#
+# When enabled, plugins are loaded automatically with the default options    #
+# when an appropriate <Plugin ...> block is encountered.                     #
+# Disabled by default.                                                       #
+#----------------------------------------------------------------------------#
+#AutoLoadPlugin false
+
+#----------------------------------------------------------------------------#
+# Interval at which to query values. This may be overwritten on a per-plugin #
+# base by using the 'Interval' option of the LoadPlugin block:               #
+#   <LoadPlugin foo>                                                         #
+#       Interval 60                                                          #
+#   </LoadPlugin>                                                            #
+#----------------------------------------------------------------------------#
+# IMPORTANT: changing this value requires a change in whisper schema as well
+Interval 20
+
+#Timeout 2
+#ReadThreads 5
+#WriteThreads 5
+
+# Limit the size of the write queue. Default is no limit. Setting up a limit
+# is recommended for servers handling a high volume of traffic.
+#WriteQueueLimitHigh 1000000
+#WriteQueueLimitLow   800000
+
+##############################################################################
+# Logging                                                                    #
+#----------------------------------------------------------------------------#
+# Plugins which provide logging functions should be loaded first, so log     #
+# messages generated when loading or configuring other plugins can be        #
+# accessed.                                                                  #
+##############################################################################
+
+LoadPlugin logfile
+#LoadPlugin syslog
+
+<Plugin logfile>
+   LogLevel "info"
+   File "/var/log/collectd.log"
+   Timestamp true
+   PrintSeverity false
+</Plugin>
+
+#<Plugin syslog>
+#    LogLevel info
+#</Plugin>
+
+##############################################################################
+# LoadPlugin section                                                         #
+#----------------------------------------------------------------------------#
+# Specify what features to activate.                                         #
+##############################################################################
+
+LoadPlugin aggregation
+#LoadPlugin amqp
+#LoadPlugin apache
+#LoadPlugin apcups
+#LoadPlugin ascent
+#LoadPlugin battery
+#LoadPlugin bind
+#LoadPlugin cgroups
+#LoadPlugin conntrack
+#LoadPlugin contextswitch
+LoadPlugin cpu
+#LoadPlugin cpufreq
+#LoadPlugin csv
+#LoadPlugin curl
+#LoadPlugin curl_json
+#LoadPlugin curl_xml
+#LoadPlugin dbi
+LoadPlugin df
+#LoadPlugin disk
+#LoadPlugin dns
+#LoadPlugin email
+#LoadPlugin entropy
+#LoadPlugin ethstat
+#LoadPlugin exec
+#LoadPlugin filecount
+#LoadPlugin fscache
+#LoadPlugin gmond
+#LoadPlugin hddtemp
+LoadPlugin interface
+#LoadPlugin ipmi
+#LoadPlugin iptables
+#LoadPlugin ipvs
+#LoadPlugin irq
+#LoadPlugin java
+#LoadPlugin libvirt
+LoadPlugin load
+#LoadPlugin lvm
+#LoadPlugin madwifi
+#LoadPlugin mbmon
+#LoadPlugin md
+#LoadPlugin memcachec
+#LoadPlugin memcached
+LoadPlugin memory
+#LoadPlugin modbus
+#LoadPlugin multimeter
+#LoadPlugin mysql
+#LoadPlugin netlink
+#LoadPlugin network
+#LoadPlugin nfs
+LoadPlugin nginx
+#LoadPlugin notify_desktop
+#LoadPlugin notify_email
+#LoadPlugin ntpd
+#LoadPlugin numa
+#LoadPlugin nut
+#LoadPlugin olsrd
+#LoadPlugin openvpn
+#<LoadPlugin perl>
+#   Globals true
+#</LoadPlugin>
+#LoadPlugin pinba
+LoadPlugin ping
+#LoadPlugin postgresql
+#LoadPlugin powerdns
+LoadPlugin processes
+#LoadPlugin protocols
+#<LoadPlugin python>
+#   Globals true
+#</LoadPlugin>
+#LoadPlugin rrdcached
+#LoadPlugin rrdtool
+#LoadPlugin sensors
+#LoadPlugin serial
+#LoadPlugin snmp
+#LoadPlugin statsd
+LoadPlugin swap
+#LoadPlugin table
+LoadPlugin tail
+#LoadPlugin tail_csv
+#LoadPlugin tcpconns
+#LoadPlugin teamspeak2
+#LoadPlugin ted
+#LoadPlugin thermal
+#LoadPlugin tokyotyrant
+#LoadPlugin unixsock
+#LoadPlugin uptime
+#LoadPlugin users
+#LoadPlugin uuid
+#LoadPlugin varnish
+LoadPlugin vmem
+#LoadPlugin vserver
+#LoadPlugin wireless
+LoadPlugin write_graphite
+#LoadPlugin write_http
+#LoadPlugin write_riemann
+
+##############################################################################
+# Plugin configuration                                                       #
+#----------------------------------------------------------------------------#
+# In this section configuration stubs for each plugin are provided. A desc-  #
+# ription of those options is available in the collectd.conf(5) manual page. #
+##############################################################################
+
+<Plugin "aggregation">
+   <Aggregation>
+       Plugin "cpu"
+       Type "cpu"
+
+       GroupBy "Host"
+       GroupBy "TypeInstance"
+
+       CalculateNum false
+       CalculateSum true
+       CalculateAverage true
+       CalculateMinimum false
+       CalculateMaximum true
+       CalculateStddev false
+   </Aggregation>
+</Plugin>
+
+<Plugin df>
+   Device "/dev/vda1"
+   Device "/dev/loop0"
+   Device "/dev/loop1"
+
+   ReportByDevice true
+   IgnoreSelected false
+
+   ValuesAbsolute true
+   ValuesPercentage true
+</Plugin>
+
+<Plugin interface>
+   Interface "eth0"
+   IgnoreSelected false
+</Plugin>
+
+<Plugin nginx>
+   URL "http://127.0.0.1/nginx_status"
+</Plugin>
+
+<Plugin ping>
+   Host "google.com"
+   Interval 1.0
+   Timeout 0.9
+   TTL 255
+</Plugin>
+
+<Plugin processes>
+   ProcessMatch "app" "node app.js"
+</Plugin>
+
+<Plugin swap>
+   ReportByDevice false
+   ReportBytes true
+</Plugin>
+
+<Plugin "tail">
+   <File "/var/log/nginx/error.log">
+     Instance "nginx"
+     <Match>
+       Regex ".*"
+       DSType "CounterInc"
+       Type counter
+       Instance "errors"
+     </Match>
+   </File>
+   <File "/var/log/nginx/access.log">
+     Instance "nginx"
+     <Match>
+       Regex ".*"
+       DSType "CounterInc"
+       Type counter
+       Instance "requests"
+     </Match>
+    <Match>
+      Regex " \".*\" [0-9]+ [0-9]+ ([0-9]+)"
+      DSType GaugeAverage
+      Type delay
+      Instance "response"
+    </Match>
+   </File>
+</Plugin>
+
+<Plugin vmem>
+   Verbose false
+</Plugin>
+
+<Plugin write_graphite>
+   <Node "graphing">
+       Host "localhost"
+       Port "2003"
+       Protocol "tcp"
+       LogSendErrors true
+       Prefix "collectd."
+       StoreRates true
+       AlwaysAppendDS false
+       EscapeCharacter "_"
+   </Node>
+</Plugin>
+
+<Include "/etc/collectd/collectd.conf.d">
+    Filter "*.conf"
+</Include>
+
--- a/setup/start/nginx/appconfig.ejs
+++ b/setup/start/nginx/appconfig.ejs
@@ -0,0 +1,115 @@
+# http://nginx.org/en/docs/http/websocket.html
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    '' close;
+}
+
+server {
+    listen       443;
+    server_name  <%= vhost %>;
+
+    ssl                  on;
+    # paths are relative to prefix and not to this file
+    ssl_certificate      cert/host.cert;
+    ssl_certificate_key  cert/host.key;
+    ssl_session_timeout  5m;
+    ssl_session_cache shared:SSL:50m;
+
+    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
+    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
+    # https://cipherli.st/
+    ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
+    ssl_ciphers 'AES128+EECDH:AES128+EDH';
+    add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
+
+    proxy_http_version 1.1;
+    proxy_intercept_errors on;
+    proxy_read_timeout       3500;
+    proxy_connect_timeout    3250;
+
+    proxy_set_header   Host               $host;
+    proxy_set_header   X-Forwarded-For    $remote_addr;
+    proxy_set_header   X-Forwarded-Host   $host;
+    proxy_set_header   X-Forwarded-Proto  https;
+
+    # upgrade is a hop-by-hop header (http://nginx.org/en/docs/http/websocket.html)
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $connection_upgrade;
+
+    error_page 500 502 503 504 =200 @appstatus;
+    location @appstatus {
+        internal;
+        root <%= sourceDir %>/webadmin/dist;
+        rewrite ^/$ /appstatus.html break;
+    }
+
+    location / {
+        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
+        proxy_buffer_size       128k;
+        proxy_buffers           4 256k;
+        proxy_busy_buffers_size 256k;
+
+        # Disable check to allow unlimited body sizes
+        client_max_body_size 0;
+
+<% if ( endpoint === 'admin' ) { %>
+        location /api/ {
+            proxy_pass   http://127.0.0.1:3000;
+            client_max_body_size 1m;
+        }
+
+        # graphite paths
+        location ~ ^/(graphite|content|metrics|dashboard|render|browser|composer)/ {
+            proxy_pass   http://127.0.0.1:8000;
+            client_max_body_size 1m;
+        }
+
+        location / {
+            root   <%= sourceDir %>/webadmin/dist;
+            index  index.html index.htm;
+        }
+
+<% } else if ( endpoint === 'oauthproxy' ) { %>
+        proxy_pass http://127.0.0.1:4000;
+        proxy_set_header   X-Cloudron-Proxy-Port   <%= port %>;
+<% } else if ( endpoint === 'app' ) { %>
+        proxy_pass http://127.0.0.1:<%= port %>;
+<% } else if ( endpoint === 'splash' ) { %>
+        root   <%= sourceDir %>;
+
+        error_page 503 /update.html;
+
+        location /update.html {
+            add_header Cache-Control no-cache;
+        }
+
+        location /theme.css {
+            add_header Cache-Control no-cache;
+        }
+
+        location /3rdparty/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /js/ {
+            add_header Cache-Control no-cache;
+        }
+
+        location /progress.json {
+            add_header Cache-Control no-cache;
+        }
+
+        location /api/v1/cloudron/progress {
+            add_header Cache-Control no-cache;
+            default_type application/json;
+            alias <%= sourceDir %>/progress.json;
+        }
+
+        location / {
+            return 503;
+        }
+<% } %>
+    }
+}
+
--- a/setup/start/nginx/mime.types
+++ b/setup/start/nginx/mime.types
@@ -0,0 +1,80 @@
+
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/x-javascript              js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg svgz;
+    image/webp                            webp;
+
+    application/java-archive              jar war ear;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/xhtml+xml                 xhtml;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              eot;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/ogg                             ogg;
+    audio/x-m4a                           m4a;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mp4                             mp4;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/webm                            webm;
+    video/x-flv                           flv;
+    video/x-m4v                           m4v;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}
--- a/setup/start/nginx/nginx.ejs
+++ b/setup/start/nginx/nginx.ejs
@@ -0,0 +1,64 @@
+user www-data;
+
+worker_processes  1;
+
+pid /run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    # the collectd config depends on this log format
+    log_format combined2 '$remote_addr - [$time_local] '
+        '"$request" $status $body_bytes_sent $request_time '
+        '"$http_referer" "$http_user_agent"';
+
+    access_log access.log combined2;
+
+    sendfile        on;
+
+    keepalive_timeout  65;
+
+    # HTTP server
+    server {
+        listen      80;
+
+        # collectd
+        location /nginx_status {
+            stub_status on;
+            access_log off;
+            allow 127.0.0.1;
+            deny all;
+        }
+
+        location / {
+            # redirect everything to HTTPS
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    # We have to enable https for nginx to read in the vhost in http request
+    # and send a 404. This is a side-effect of using wildcard DNS
+    server {
+        listen 443 default_server;
+        ssl on;
+        ssl_certificate      cert/host.cert;
+        ssl_certificate_key  cert/host.key;
+
+        error_page 404 = @fallback;
+        location @fallback {
+            internal;
+            root <%= sourceDir %>/webadmin/dist;
+            rewrite ^/$ /nakeddomain.html break;
+        }
+
+        return 404;
+    }
+
+    include applications/*.conf;
+}
+
--- a/setup/start/setup_infra.sh
+++ b/setup/start/setup_infra.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+readonly DATA_DIR="/home/yellowtent/data"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
+
+arg_fqdn="$1"
+
+# removing containers ensures containers are launched with latest config updates
+# restore code in appatask does not delete old containers
+infra_version="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
+if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
+    echo "Infrastructure is upto date"
+    exit 0
+fi
+
+echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
+
+existing_containers=$(docker ps -qa)
+echo "Remove containers: ${existing_containers}"
+if [[ -n "${existing_containers}" ]]; then
+    echo "${existing_containers}" | xargs docker rm -f
+fi
+
+# graphite
+docker run --restart=always -d --name="graphite" \
+    -p 127.0.0.1:2003:2003 \
+    -p 127.0.0.1:2004:2004 \
+    -p 127.0.0.1:8000:8000 \
+    -v "${DATA_DIR}/graphite:/app/data" cloudron/graphite:0.3.1
+
+# mail
+mail_container_id=$(docker run --restart=always -d --name="mail" \
+    -p 127.0.0.1:25:25 \
+    -h "${arg_fqdn}" \
+    -e "DOMAIN_NAME=${arg_fqdn}" \
+    -v "${DATA_DIR}/box/mail:/app/data" \
+    cloudron/mail:0.3.0)
+echo "Mail container id: ${mail_container_id}"
+
+# mysql
+mysql_addon_root_password=$(pwgen -1 -s)
+docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
+cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
+readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
+readonly MYSQL_ROOT_HOST='${docker0_ip}'
+EOF
+mysql_container_id=$(docker run --restart=always -d --name="mysql" \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
+    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:r" \
+    cloudron/mysql:0.3.0)
+echo "MySQL container id: ${mysql_container_id}"
+
+# postgresql
+postgresql_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
+readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
+EOF
+postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
+    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:r" \
+    cloudron/postgresql:0.3.0)
+echo "PostgreSQL container id: ${postgresql_container_id}"
+
+# mongodb
+mongodb_addon_root_password=$(pwgen -1 -s)
+cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
+readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
+EOF
+mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
+    -h "${arg_fqdn}" \
+    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
+    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:r" \
+    cloudron/mongodb:0.3.0)
+echo "Mongodb container id: ${mongodb_container_id}"
+
+if [[ "${infra_version}" == "none" ]]; then
+    # if no existing infra was found (for new and restoring cloudons), download app backups
+    echo "Marking installed apps for restore"
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore" WHERE installationState = "installed"' box
+else
+    # if existing infra was found, just mark apps for reconfiguration
+    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure" WHERE installationState = "installed"' box
+fi
+
+echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"
+
--- a/setup/stop.sh
+++ b/setup/stop.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+echo "Stopping box code"
+
+service supervisor stop || true
+
+echo -n "Waiting for supervisord to stop"
+while test -e "/var/run/supervisord.pid" && kill -0 `cat /var/run/supervisord.pid`; do
+    echo -n "."
+    sleep 1
+done
+echo ""
+