Add changes

Fixes #682

.eslintrc.json

@@ -0,0 +1,29 @@
+{
+    "env": {
+        "node": true,
+        "es6": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 8
+    },
+    "rules": {
+        "indent": [
+            "error",
+            4
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "quotes": [
+            "error",
+            "single"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ],
+        "no-console": "off"
+    }
+}

.gitattributes

@@ -1,7 +1,7 @@
 # following files are skipped when exporting using git archive
-/release export-ignore
-/admin export-ignore
 test export-ignore
+.jshintrc export-ignore
+.gitlab export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
 

.gitignore

@@ -1,11 +1,8 @@
 node_modules/
 coverage/
-docs/
 webadmin/dist/
-setup/splash/website/
 installer/src/certs/server.key
 
 # vim swap files
 *.swp
 
-

.gitlab/issue_templates/Bug.md

@@ -0,0 +1,6 @@
+Please do not use this issue tracker for support requests and bug reports.
+This issue tracker is used by the Cloudron development team to track actual
+bugs in the code.
+
+Please use the forum at https://forum.cloudron.io to report bugs. For
+confidential issues, please email us at support@cloudron.io.

.gitlab/issue_templates/Feature.md

@@ -0,0 +1,7 @@
+Please do not use this issue tracker for support requests and feature reports.
+This issue tracker is used by the Cloudron development team to track issues in
+the code.
+
+Please use the forum at https://forum.cloudron.io to report bugs. For
+confidential issues, please email us at support@cloudron.io.
+

.jshintrc

@@ -1,7 +0,0 @@
-{
-	"node": true,
-	"browser": true,
-	"unused": true,
-	"globalstrict": true,
-	"predef": [ "angular", "$" ]
-}

LICENSE

@@ -0,0 +1,35 @@
+The Cloudron Subscription license
+Copyright (c) 2020 Cloudron UG
+
+With regard to the Cloudron Software:
+
+This software and associated documentation files (the "Software") may only be
+used in production, if you (and any entity that you represent) have agreed to,
+and are in compliance with, the Cloudron Subscription Terms of Service, available
+at https://cloudron.io/legal/terms.html (the “Subscription Terms”), or other
+agreement governing the use of the Software, as agreed by you and Cloudron,
+and otherwise have a valid Cloudron Subscription. Subject to the foregoing sentence,
+you are free to modify this Software and publish patches to the Software. You agree
+that Subscription and/or its licensors (as applicable) retain all right, title and
+interest in and to all such modifications and/or patches, and all such modifications
+and/or patches may only be used, copied, modified, displayed, distributed, or otherwise
+exploited with a valid Cloudron subscription. Notwithstanding the foregoing, you may copy
+and modify the Software for development and testing purposes, without requiring a
+subscription.  You agree that Cloudron and/or its licensors (as applicable) retain
+all right, title and interest in and to all such modifications.  You are not
+granted any other rights beyond what is expressly stated herein.  Subject to the
+foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
+and/or sell the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+For all third party components incorporated into the Cloudron Software, those
+components are licensed under the original license provided by the owner of the
+applicable component.
+

README.md

@@ -1,17 +1,55 @@
-Cloudron a Smart Server
-=======================
+# Cloudron
 
+[Cloudron](https://cloudron.io) is the best way to run apps on your server.
 
+Web applications like email, contacts, blog, chat are the backbone of the modern
+internet. Yet, we live in a world where hosting these essential applications is
+a complex task.
 
-Selfhost Instructions
----------------------
+We are building the ultimate platform for self-hosting web apps. The Cloudron allows
+anyone to effortlessly host web applications on their server on their own terms.
 
-The smart server currently relies on an AWS account with access to Route53 and S3 and is tested on DigitalOcean and EC2.
+## Features
 
-First create a virtual private server with Ubuntu 15.04 and run the following commands in an ssh session to initialize the base image:
+* Single click install for apps. Check out the [App Store](https://cloudron.io/appstore.html).
+
+* Per-app encrypted backups and restores.
+
+* App updates delivered via the App Store.
+
+* Secure - Cloudron manages the firewall. All apps are secured with HTTPS. Certificates are
+  installed and renewed automatically.
+
+* Centralized User & Group management. Control who can access which app.
+
+* Single Sign On. Use same credentials across all apps.
+
+* Automatic updates for the Cloudron platform.
+
+* Trivially migrate to another server keeping your apps and data (for example, switch your
+  infrastructure provider or move to a bigger server).
+
+* Comprehensive [REST API](https://cloudron.io/documentation/developer/api/).
+
+* [CLI](https://cloudron.io/documentation/cli/) to configure apps.
+
+* Alerts, audit logs, graphs, dns management ... and much more
+
+## Demo
+
+Try our demo at https://my.demo.cloudron.io (username: cloudron password: cloudron).
+
+## Installing
+
+[Install script](https://cloudron.io/documentation/installation/) - [Pricing](https://cloudron.io/pricing.html)
+
+**Note:** This repo is a small part of what gets installed on your server - there is
+the dashboard, database addons, graph container, base image etc. Cloudron also relies
+on external services such as the App Store for apps to be installed. As such, don't
+clone this repo and npm install and expect something to work.
+
+## Support
+
+* [Documentation](https://cloudron.io/documentation/)
+* [Forum](https://forum.cloudron.io/)
 
-```
-curl https://s3.amazonaws.com/prod-cloudron-releases/installer.sh -o installer.sh
-chmod +x installer.sh
-./installer.sh <domain> <aws access key> <aws acccess secret> <backup bucket> <provider> <release sha1>
-```

VERSION

@@ -0,0 +1 @@
+# release version. do not edit manually

baseimage/createAMI

@@ -0,0 +1,193 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+assertNotEmpty() {
+    : "${!1:? "$1 is not set."}"
+}
+
+readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+export JSON="${SOURCE_DIR}/node_modules/.bin/json"
+
+INSTANCE_TYPE="t2.micro"
+BLOCK_DEVICE="DeviceName=/dev/sda1,Ebs={VolumeSize=20,DeleteOnTermination=true,VolumeType=gp2}"
+SSH_KEY_NAME="id_rsa_yellowtent"
+
+revision=$(git rev-parse HEAD)
+ami_name=""
+server_id=""
+server_ip=""
+destroy_server="yes"
+deploy_env="prod"
+image_id=""
+
+args=$(getopt -o "" -l "revision:,name:,no-destroy,env:,region:" -n "$0" -- "$@")
+eval set -- "${args}"
+
+while true; do
+    case "$1" in
+    --env) deploy_env="$2"; shift 2;;
+    --revision) revision="$2"; shift 2;;
+    --name) ami_name="$2"; shift 2;;
+    --no-destroy) destroy_server="no"; shift 2;;
+    --region)
+        case "$2" in
+        "us-east-1")
+            image_id="ami-6edd3078"
+            security_group="sg-a5e17fd9"
+            subnet_id="subnet-b8fbc0f1"
+            ;;
+        "eu-central-1")
+            image_id="ami-5aee2235"
+            security_group="sg-19f5a770" # everything open on eu-central-1
+            subnet_id=""
+            ;;
+        *)
+            echo "Unknown aws region $2"
+            exit 1
+            ;;
+        esac
+        export AWS_DEFAULT_REGION="$2"    # used by the aws cli tool
+        shift 2
+        ;;
+    --) break;;
+    *) echo "Unknown option $1"; exit 1;;
+    esac
+done
+
+# TODO fix this
+export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY}"
+export AWS_SECRET_ACCESS_KEY="${AWS_ACCESS_SECRET}"
+
+readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent"
+readonly SSH="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
+
+if [[ ! -f "${ssh_keys}" ]]; then
+    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
+    exit 1
+fi
+
+if [[ -z "${image_id}" ]]; then
+    echo "--region is required (us-east-1 or eu-central-1)"
+    exit 1
+fi
+
+function get_pretty_revision() {
+    local git_rev="$1"
+    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
+
+    echo "${sha1}"
+}
+
+function wait_for_ssh() {
+    echo "=> Waiting for ssh connection"
+    while true; do
+        echo -n "."
+
+        if $SSH ubuntu@${server_ip} echo "hello"; then
+            echo ""
+            break
+        fi
+
+        sleep 5
+    done
+}
+
+now=$(date "+%Y-%m-%d-%H%M%S")
+pretty_revision=$(get_pretty_revision "${revision}")
+
+if [[ -z "${ami_name}" ]]; then
+    ami_name="box-${deploy_env}-${pretty_revision}-${now}"
+fi
+
+echo "=> Create EC2 instance"
+id=$(aws ec2 run-instances --image-id "${image_id}" --instance-type "${INSTANCE_TYPE}" --security-group-ids "${security_group}" --block-device-mappings "${BLOCK_DEVICE}" --key-name "${SSH_KEY_NAME}" --subnet-id "${subnet_id}" --associate-public-ip-address \
+    | $JSON Instances \
+    | $JSON 0.InstanceId)
+
+[[ -z "$id" ]] && exit 1
+echo "Instance created  ID $id"
+
+echo "=> Waiting for instance to get a public IP"
+while true; do
+    server_ip=$(aws ec2 describe-instances --instance-ids ${id} \
+        | $JSON Reservations.0.Instances \
+        | $JSON 0.PublicIpAddress)
+
+    if [[ ! -z "${server_ip}" ]]; then
+        echo ""
+        break
+    fi
+
+    echo -n "."
+    sleep 1
+done
+
+echo "Got public IP ${server_ip}"
+
+wait_for_ssh
+
+echo "=> Fetching cloudron-setup"
+while true; do
+
+    if $SSH ubuntu@${server_ip} wget "https://cloudron.io/cloudron-setup" -O "cloudron-setup"; then
+        echo ""
+        break
+    fi
+
+    echo -n "."
+    sleep 5
+done
+
+echo "=> Running cloudron-setup"
+$SSH ubuntu@${server_ip} sudo /bin/bash "cloudron-setup" --env "${deploy_env}" --provider "ami" --skip-reboot
+
+wait_for_ssh
+
+echo "=> Removing ssh key"
+$SSH ubuntu@${server_ip} sudo rm /home/ubuntu/.ssh/authorized_keys /root/.ssh/authorized_keys
+
+echo "=> Creating AMI"
+image_id=$(aws ec2 create-image --instance-id "${id}" --name "${ami_name}" | $JSON ImageId)
+[[ -z "$id" ]] && exit 1
+echo "Creating AMI with Id ${image_id}"
+
+echo "=> Waiting for AMI to be created"
+while true; do
+    state=$(aws ec2 describe-images --image-ids ${image_id} \
+        | $JSON Images \
+        | $JSON 0.State)
+
+    if [[ "${state}" == "available" ]]; then
+        echo ""
+        break
+    fi
+
+    echo -n "."
+    sleep 5
+done
+
+if [[ "${destroy_server}" == "yes" ]]; then
+    echo "=> Deleting EC2 instance"
+
+    while true; do
+        state=$(aws ec2 terminate-instances --instance-id "${id}" \
+            | $JSON TerminatingInstances \
+            | $JSON 0.CurrentState.Name)
+
+        if [[ "${state}" == "shutting-down" ]]; then
+            echo ""
+            break
+        fi
+
+        echo -n "."
+        sleep 5
+    done
+fi
+
+echo ""
+echo "Done."
+echo ""
+echo "New AMI is: ${image_id}"
+echo ""

baseimage/createImage

@@ -1,186 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-assertNotEmpty() {
-    : "${!1:? "$1 is not set."}"
-}
-
-readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
-export JSON="${SOURCE_DIR}/node_modules/.bin/json"
-
-provider="digitalocean"
-installer_revision=$(git rev-parse HEAD)
-box_name=""
-server_id=""
-server_ip=""
-destroy_server="yes"
-deploy_env="dev"
-
-# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt
-# brew install gnu-getopt to get the GNU getopt on OS X
-[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt"
-readonly GNU_GETOPT
-
-args=$(${GNU_GETOPT} -o "" -l "provider:,revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --env) deploy_env="$2"; shift 2;;
-    --revision) installer_revision="$2"; shift 2;;
-    --provider) provider="$2"; shift 2;;
-    --name) box_name="$2"; destroy_server="no"; shift 2;;
-    --no-destroy) destroy_server="no"; shift 2;;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-done
-
-echo "Creating image using ${provider}"
-if [[ "${provider}" == "digitalocean" ]]; then
-    if [[ "${deploy_env}" == "staging" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}"
-    elif [[ "${deploy_env}" == "dev" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}"
-    elif [[ "${deploy_env}" == "prod" ]]; then
-        assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD
-        export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}"
-    else
-        echo "No such env ${deploy_env}."
-        exit 1
-    fi
-
-    vps="/bin/bash ${SCRIPT_DIR}/digitalocean.sh"
-else
-    echo "Unknown provider : ${provider}"
-    exit 1
-fi
-
-readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}"
-readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-
-readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}"
-
-if [[ ! -f "${ssh_keys}" ]]; then
-    echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)"
-    exit 1
-fi
-
-function get_pretty_revision() {
-    local git_rev="$1"
-    local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null)
-
-    echo "${sha1}"
-}
-
-now=$(date "+%Y-%m-%d-%H%M%S")
-pretty_revision=$(get_pretty_revision "${installer_revision}")
-
-if [[ -z "${box_name}" ]]; then
-    # if you change this, change the regexp is appstore/janitor.js
-    box_name="box-${deploy_env}-${pretty_revision}-${now}" # remove slashes
-
-    # create a new server if no name given
-    if ! caas_ssh_key_id=$($vps get_ssh_key_id "caas"); then
-        echo "Could not query caas ssh key"
-        exit 1
-    fi
-    echo "Detected caas ssh key id: ${caas_ssh_key_id}"
-
-    echo "Creating Server with name [${box_name}]"
-    if ! server_id=$($vps create ${caas_ssh_key_id} ${box_name}); then
-        echo "Failed to create server"
-        exit 1
-    fi
-    echo "Created server with id: ${server_id}"
-
-    # If we run scripts overenthusiastically without the wait, setup script randomly fails
-    echo -n "Waiting 120 seconds for server creation"
-    for i in $(seq 1 24); do
-        echo -n "."
-        sleep 5
-    done
-    echo ""
-else
-    if ! server_id=$($vps get_id "${box_name}"); then
-        echo "Could not determine id from name"
-        exit 1
-    fi
-    echo "Reusing server with id: ${server_id}"
-
-    $vps power_on "${server_id}"
-fi
-
-# Query until we get an IP
-while true; do
-    echo "Trying to get the server IP"
-    if server_ip=$($vps get_ip "${server_id}"); then
-        echo "Server IP : [${server_ip}]"
-        break
-    fi
-    echo "Timedout, trying again in 10 seconds"
-    sleep 10
-done
-
-while true; do
-    echo "Trying to copy init script to server"
-    if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" root@${server_ip}:.; then
-        break
-    fi
-    echo "Timedout, trying again in 30 seconds"
-    sleep 30
-done
-
-echo "Copying INFRA_VERSION"
-$scp22 "${SCRIPT_DIR}/../setup/INFRA_VERSION" root@${server_ip}:.
-
-echo "Copying box source"
-cd "${SOURCE_DIR}"
-git archive --format=tar HEAD | $ssh22 "root@${server_ip}" "cat - > /tmp/box.tar.gz"
-
-echo "Executing init script"
-if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then
-    echo "Init script failed"
-    exit 1
-fi
-
-echo "Shutting down server with id : ${server_id}"
-$ssh202 "root@${server_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail
-
-# wait 10 secs for actual shutdown
-echo "Waiting for 10 seconds for server to shutdown"
-sleep 30
-
-echo "Powering off server"
-if ! $vps power_off "${server_id}"; then
-    echo "Could not power off server"
-    exit 1
-fi
-
-snapshot_name="box-${deploy_env}-${pretty_revision}-${now}"
-echo "Snapshotting as ${snapshot_name}"
-if ! image_id=$($vps snapshot "${server_id}" "${snapshot_name}"); then
-    echo "Could not snapshot and get image id"
-    exit 1
-fi
-
-if [[ "${destroy_server}" == "yes" ]]; then
-    echo "Destroying server"
-    if ! $vps destroy "${server_id}"; then
-        echo "Could not destroy server"
-        exit 1
-    fi
-else
-    echo "Skipping server destroy"
-fi
-
-echo "Transferring image ${image_id} to other regions"
-$vps transfer_image_to_all_regions "${image_id}"
-
-echo "Done."

baseimage/digitalocean.sh

@@ -10,7 +10,7 @@ if [[ -z "${JSON}" ]]; then
     exit 1
 fi
 
-readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
 
 function debug() {
     echo "$@" >&2
@@ -29,9 +29,9 @@ function create_droplet() {
     local ssh_key_id="$1"
     local box_name="$2"
 
-    local image_region="sfo1"
-    local ubuntu_image_slug="ubuntu-15-10-x64"
-    local box_size="512mb"
+    local image_region="sfo2"
+    local ubuntu_image_slug="ubuntu-16-04-x64"
+    local box_size="1gb"
 
     local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}"
 
@@ -49,9 +49,9 @@ function get_droplet_ip() {
 
 function get_droplet_id() {
     local droplet_name="$1"
-    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
     [[ -z "$id" ]] && exit 1
-    echo "$id"  
+    echo "$id"
 }
 
 function power_off_droplet() {
@@ -109,13 +109,24 @@ function get_image_id() {
     local snapshot_name="$1"
     local image_id=""
 
-    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
-
-    if [[ -n "${image_id}" ]]; then
-        echo "${image_id}"
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
     fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+        return 1
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
 }
 
 function snapshot_droplet() {
@@ -128,16 +139,26 @@ function snapshot_droplet() {
     debug -n "Waiting for snapshot to complete"
 
     while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
         if [[ "${event_status}" == "completed" ]]; then
             break
         fi
         debug -n "."
         sleep 10
     done
-    debug ""
+    debug "! done"
 
-    get_image_id "${snapshot_name}"
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
 }
 
 function destroy_droplet() {
@@ -177,7 +198,7 @@ function transfer_image_to_all_regions() {
     local image_id="$1"
 
     xfer_events=()
-    image_regions=(ams3) ## sfo1 is where the image is created
+    image_regions=(ams2) ## sfo1 is where the image is created
     for image_region in ${image_regions[@]}; do
         xfer_event=$(transfer_image ${image_id} ${image_region})
         echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}"

baseimage/initializeBaseUbuntuImage.sh

@@ -2,325 +2,163 @@
 
 set -euv -o pipefail
 
-readonly USER=yellowtent
-readonly USER_HOME="/home/${USER}"
-readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer"
-readonly INSTALLER_REVISION="$1"
-readonly SELFHOSTED=$(( $# > 1 ? 1 : 0 ))
-readonly USER_DATA_FILE="/root/user_data.img"
-readonly USER_DATA_DIR="/home/yellowtent/data"
-
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
+readonly arg_provider="${1:-generic}"
+readonly arg_infraversionpath="${SOURCE_DIR}/${2:-}"
+
 function die {
     echo $1
     exit 1
 }
 
-[[ "$(systemd --version 2>&1)" == *"systemd 225"* ]] || die "Expecting systemd to be 225"
-
-if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
-    source "${SOURCE_DIR}/INFRA_VERSION"
-else
-    echo "No INFRA_VERSION found, skip pulling docker images"
-fi
-
-if [ ${SELFHOSTED} == 0 ]; then
-    echo "!! Initializing Ubuntu image for CaaS"
-else
-    echo "!! Initializing Ubuntu image for Selfhosting"
-fi
-
-echo "==== Create User ${USER} ===="
-if ! id "${USER}"; then
-    useradd "${USER}" -m
-fi
-
-echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ==="
-
-echo "=== Prepare installer source ==="
-rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}"
-rm -rf /tmp/box && mkdir -p /tmp/box
-tar xvf /tmp/box.tar.gz -C /tmp/box && rm /tmp/box.tar.gz
-cp -rf /tmp/box/installer/* "${INSTALLER_SOURCE_DIR}"
-echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION"
-
 export DEBIAN_FRONTEND=noninteractive
 
-echo "=== Upgrade ==="
-apt-get update
-apt-get dist-upgrade -y
-apt-get install -y curl
+# hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
+apt-mark hold grub* >/dev/null
+apt-get -o Dpkg::Options::="--force-confdef" update -y
+apt-get -o Dpkg::Options::="--force-confdef" upgrade -y
+apt-mark unhold grub* >/dev/null
 
-# Setup firewall before everything. docker creates it's own chain and the -X below will remove it
-# Do NOT use iptables-persistent because it's startup ordering conflicts with docker
-echo "=== Setting up firewall ==="
-# clear tables and set default policy
-iptables -F # flush all chains
-iptables -X # delete all chains
-# default policy for filter table
-iptables -P INPUT DROP
-iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject
-iptables -P OUTPUT ACCEPT
+echo "==> Installing required packages"
 
-# NOTE: keep these in sync with src/apps.js validatePortBindings
-# allow ssh, http, https, ping, dns
-iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-if [ ${SELFHOSTED} == 0 ]; then
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
-else
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
-fi
-iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
-iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
-iptables -A INPUT -p udp --sport 53 -j ACCEPT
-iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
-
-# loopback
-iptables -A INPUT -i lo -j ACCEPT
-iptables -A OUTPUT -o lo -j ACCEPT
-
-# prevent DoS
-# iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT
-
-# log dropped incoming. keep this at the end of all the rules
-iptables -N LOGGING # new chain
-iptables -A INPUT -j LOGGING # last rule in INPUT chain
-iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7
-iptables -A LOGGING -j DROP
-
-echo "==== Install btrfs tools ==="
-apt-get -y install btrfs-tools
-
-echo "==== Install docker ===="
-# install docker from binary to pin it to a specific version. the current debian repo does not allow pinning
-curl https://get.docker.com/builds/Linux/x86_64/docker-1.10.2 > /usr/bin/docker
-apt-get -y install aufs-tools
-chmod +x /usr/bin/docker
-groupadd docker
-cat > /etc/systemd/system/docker.socket <<EOF
-[Unit]
-Description=Docker Socket for the API
-PartOf=docker.service
-
-[Socket]
-ListenStream=/var/run/docker.sock
-SocketMode=0660
-SocketUser=root
-SocketGroup=docker
-
-[Install]
-WantedBy=sockets.target
-EOF
-cat > /etc/systemd/system/docker.service <<EOF
-[Unit]
-Description=Docker Application Container Engine
-After=network.target docker.socket
-Requires=docker.socket
-
-[Service]
-ExecStart=/usr/bin/docker daemon -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-echo "=== Setup btrfs data ==="
-truncate -s "8192m" "${USER_DATA_FILE}" # 8gb start (this will get resized dynamically by box-setup.service)
-mkfs.btrfs -L UserHome "${USER_DATA_FILE}"
-mkdir -p "${USER_DATA_DIR}"
-mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
-
-systemctl daemon-reload
-systemctl enable docker
-systemctl start docker
-
-# give docker sometime to start up and create iptables rules
-# those rules come in after docker has started, and we want to wait for them to be sure iptables-save has all of them
-sleep 10
-
-# Disable forwarding to metadata route from containers
-iptables -I FORWARD -d 169.254.169.254 -j DROP
-
-# ubuntu will restore iptables from this file automatically. this is here so that docker's chain is saved to this file
-mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4
-
-echo "=== Enable memory accounting =="
-sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
-update-grub
-
-# now add the user to the docker group
-usermod "${USER}" -a -G docker
-
-if [ -z $(echo "${INFRA_VERSION}") ]; then
-    echo "Skip pulling base docker images"
-else
-    echo "=== Pulling base docker images ==="
-    docker pull "${BASE_IMAGE}"
-
-    echo "=== Pulling mysql addon image ==="
-    docker pull "${MYSQL_IMAGE}"
-
-    echo "=== Pulling postgresql addon image ==="
-    docker pull "${POSTGRESQL_IMAGE}"
-
-    echo "=== Pulling redis addon image ==="
-    docker pull "${REDIS_IMAGE}"
-
-    echo "=== Pulling mongodb addon image ==="
-    docker pull "${MONGODB_IMAGE}"
-
-    echo "=== Pulling graphite docker images ==="
-    docker pull "${GRAPHITE_IMAGE}"
-
-    echo "=== Pulling mail relay ==="
-    docker pull "${MAIL_IMAGE}"
-fi
-
-echo "==== Install nginx ===="
-apt-get -y install nginx-full
-[[ "$(nginx -v 2>&1)" == *"nginx/1.9."* ]] || die "Expecting nginx version to be 1.9.x"
-
-echo "==== Install build-essential ===="
-apt-get -y install build-essential rcconf
-
-echo "==== Install mysql ===="
 debconf-set-selections <<< 'mysql-server mysql-server/root_password password password'
 debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password'
-apt-get -y install mysql-server
-[[ "$(mysqld --version 2>&1)" == *"5.6."* ]] || die "Expecting nginx version to be 5.6.x"
 
-echo "==== Install pwgen and swaks awscli ===="
-apt-get -y install pwgen swaks awscli
+# this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
+# resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
+ubuntu_version=$(lsb_release -rs)
+ubuntu_codename=$(lsb_release -cs)
+gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
+apt-get -y install \
+    acl \
+    build-essential \
+    cifs-utils \
+    cron \
+    curl \
+    debconf-utils \
+    dmsetup \
+    $gpg_package \
+    iptables \
+    libpython2.7 \
+    linux-generic \
+    logrotate \
+    mysql-server-5.7 \
+    openssh-server \
+    pwgen \
+    resolvconf \
+    swaks \
+    tzdata \
+    unattended-upgrades \
+    unbound \
+    xfsprogs
 
-echo "==== Install collectd ==="
-if ! apt-get install -y collectd collectd-utils; then
+if [[ "${ubuntu_version}" == "16.04" ]]; then
+    echo "==> installing nginx for xenial for TLSv3 support"
+
+    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.14.0-1~xenial_amd64.deb -o /tmp/nginx.deb
+    # apt install with install deps (as opposed to dpkg -i)
+    apt install -y /tmp/nginx.deb
+    rm /tmp/nginx.deb
+else
+    apt install -y nginx-full
+fi
+
+# on some providers like scaleway the sudo file is changed and we want to keep the old one
+apt-get -o Dpkg::Options::="--force-confold" install -y sudo
+
+# this ensures that unattended upgades are enabled, if it was disabled during ubuntu install time (see #346)
+# debconf-set-selection of unattended-upgrades/enable_auto_updates + dpkg-reconfigure does not work
+cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades
+
+echo "==> Installing node.js"
+mkdir -p /usr/local/node-10.18.1
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
+ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
+ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
+
+# https://docs.docker.com/engine/installation/linux/ubuntulinux/
+echo "==> Installing Docker"
+
+# create systemd drop-in file. if you channge options here, be sure to fixup installer.sh as well
+mkdir -p /etc/systemd/system/docker.service.d
+echo -e "[Service]\nExecStart=\nExecStart=/usr/bin/dockerd -H fd:// --log-driver=journald --exec-opt native.cgroupdriver=cgroupfs --storage-driver=overlay2" > /etc/systemd/system/docker.service.d/cloudron.conf
+
+# there are 3 packages for docker - containerd, CLI and the daemon
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/containerd.io_1.2.2-3_amd64.deb" -o /tmp/containerd.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce-cli_18.09.2~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker-ce-cli.deb
+curl -sL "https://download.docker.com/linux/ubuntu/dists/${ubuntu_codename}/pool/stable/amd64/docker-ce_18.09.2~3-0~ubuntu-${ubuntu_codename}_amd64.deb" -o /tmp/docker.deb
+# apt install with install deps (as opposed to dpkg -i)
+apt install -y /tmp/containerd.deb  /tmp/docker-ce-cli.deb /tmp/docker.deb
+rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
+
+storage_driver=$(docker info | grep "Storage Driver" | sed 's/.*: //')
+if [[ "${storage_driver}" != "overlay2" ]]; then
+    echo "Docker is using "${storage_driver}" instead of overlay2"
+    exit 1
+fi
+
+# do not upgrade grub because it might prompt user and break this script
+echo "==> Enable memory accounting"
+apt-get -y --no-upgrade install grub2-common
+sed -e 's/^GRUB_CMDLINE_LINUX="\(.*\)"$/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub
+update-grub
+
+echo "==> Downloading docker images"
+if [ ! -f "${arg_infraversionpath}/infra_version.js" ]; then
+    echo "No infra_versions.js found"
+    exit 1
+fi
+
+images=$(node -e "var i = require('${arg_infraversionpath}/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
+
+echo -e "\tPulling docker images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+    docker pull "${image%@sha256:*}" # this will tag the image for readability
+done
+
+echo "==> Install collectd"
+if ! apt-get install -y libcurl3-gnutls collectd collectd-utils; then
     # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
     echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
     sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf
 fi
-update-rc.d -f collectd remove
 
-# this simply makes it explicit that we run logrotate via cron. it's already part of base ubuntu
-echo "==== Install logrotate ==="
-apt-get install -y cron logrotate
-systemctl enable cron
-
-echo "==== Install nodejs ===="
-# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
-mkdir -p /usr/local/node-4.1.1
-curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
-ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
-ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
-echo "=== Rebuilding npm packages ==="
-cd "${INSTALLER_SOURCE_DIR}" && npm install --production
-chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"
-
-echo "==== Install installer systemd script ===="
-provisionEnv="PROVISION=digitalocean"
-if [ ${SELFHOSTED} == 1 ]; then
-    provisionEnv="PROVISION=local"
-fi
-
-cat > /etc/systemd/system/cloudron-installer.service <<EOF
-[Unit]
-Description=Cloudron Installer
-; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
-BindsTo=systemd-journald.service
-
-[Service]
-Type=idle
-ExecStart="${INSTALLER_SOURCE_DIR}/src/server.js"
-Environment="DEBUG=installer*,connect-lastmile" ${provisionEnv}
-; kill any child (installer.sh) as well
-KillMode=control-group
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-# Restore iptables before docker
-echo "==== Install iptables-restore systemd script ===="
-cat > /etc/systemd/system/iptables-restore.service <<EOF
-[Unit]
-Description=IPTables Restore
-Before=docker.service
-
-[Service]
-Type=oneshot
-ExecStart=/sbin/iptables-restore /etc/iptables/rules.v4
-RemainAfterExit=yes
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-# Allocate swap files
-# https://bbs.archlinux.org/viewtopic.php?id=194792 ensures this runs after do-resize.service
-echo "==== Install box-setup systemd script ===="
-cat > /etc/systemd/system/box-setup.service <<EOF
-[Unit]
-Description=Box Setup
-Before=docker.service collectd.service mysql.service
-After=do-resize.service
-
-[Service]
-Type=oneshot
-ExecStart="${INSTALLER_SOURCE_DIR}/systemd/box-setup.sh"
-RemainAfterExit=yes
-
-[Install]
-WantedBy=multi-user.target
-EOF
-
-systemctl daemon-reload
-systemctl enable cloudron-installer
-systemctl enable iptables-restore
-systemctl enable box-setup
-
-# Configure systemd
-sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
-    -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
-    -i /etc/systemd/journald.conf
-
-# When rotating logs, systemd kills journald too soon sometimes
-# See https://github.com/systemd/systemd/issues/1353 (this is upstream default)
-sed -e "s/^WatchdogSec=.*$/WatchdogSec=3min/" \
-    -i /lib/systemd/system/systemd-journald.service
-
-sync
-
-# Configure time
+echo "==> Configuring host"
 sed -e 's/^#NTP=/NTP=0.ubuntu.pool.ntp.org 1.ubuntu.pool.ntp.org 2.ubuntu.pool.ntp.org 3.ubuntu.pool.ntp.org/' -i /etc/systemd/timesyncd.conf
 timedatectl set-ntp 1
+# mysql follows the system timezone
 timedatectl set-timezone UTC
 
-# Give user access to system logs
-apt-get -y install acl
-usermod -a -G systemd-journal ${USER}
-mkdir -p /var/log/journal  # in some images, this directory is not created making system log to /run/systemd instead
-chown root:systemd-journal /var/log/journal
-systemctl restart systemd-journald
-setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
+echo "==> Adding sshd configuration warning"
+sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config
 
-if [ ${SELFHOSTED} == 0 ]; then
-    echo "==== Install ssh ==="
-    apt-get -y install openssh-server
-    # https://stackoverflow.com/questions/4348166/using-with-sed on why ? must be escaped
-    sed -e 's/^#\?Port .*/Port 202/g' \
-        -e 's/^#\?PermitRootLogin .*/PermitRootLogin without-password/g' \
-        -e 's/^#\?PermitEmptyPasswords .*/PermitEmptyPasswords no/g' \
-        -e 's/^#\?PasswordAuthentication .*/PasswordAuthentication no/g' \
-        -i /etc/ssh/sshd_config
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+
+# Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
+systemctl stop bind9 || true
+systemctl disable bind9 || true
+
+# on ovh images dnsmasq seems to run by default
+systemctl stop dnsmasq || true
+systemctl disable dnsmasq || true
+
+# on ssdnodes postfix seems to run by default
+systemctl stop postfix || true
+systemctl disable postfix || true
+
+# on ubuntu 18.04, this is the default. this requires resolvconf for DNS to work further after the disable
+systemctl stop systemd-resolved || true
+systemctl disable systemd-resolved || true
+
+# ubuntu's default config for unbound does not work if ipv6 is disabled. this config is overwritten in start.sh
+# we need unbound to work as this is required for installer.sh to do any DNS requests
+ip6=$([[ -s /proc/net/if_inet6 ]] && echo "yes" || echo "no")
+echo -e "server:\n\tinterface: 127.0.0.1\n\tdo-ip6: ${ip6}" > /etc/unbound/unbound.conf.d/cloudron-network.conf
+systemctl restart unbound
 
-    # required so we can connect to this machine since port 22 is blocked by iptables by now
-    systemctl reload sshd
-fi

box.js

@@ -2,63 +2,57 @@
 
 'use strict';
 
+// prefix all output with a timestamp
+// debug() already prefixes and uses process.stderr NOT console.*
+['log', 'info', 'warn', 'debug', 'error'].forEach(function (log) {
+    var orig = console[log];
+    console[log] = function () {
+        orig.apply(console, [new Date().toISOString()].concat(Array.prototype.slice.call(arguments)));
+    };
+});
+
 require('supererror')({ splatchError: true });
 
-// remove timestamp from debug() based output
-require('debug').formatArgs = function formatArgs() {
-    arguments[0] = this.namespace + ' ' + arguments[0];
-    return arguments;
-};
-
-var appHealthMonitor = require('./src/apphealthmonitor.js'),
-    async = require('async'),
-    config = require('./src/config.js'),
+let async = require('async'),
+    constants = require('./src/constants.js'),
+    dockerProxy = require('./src/dockerproxy.js'),
     ldap = require('./src/ldap.js'),
-    oauthproxy = require('./src/oauthproxy.js'),
-    server = require('./src/server.js'),
-    simpleauth = require('./src/simpleauth.js');
+    server = require('./src/server.js');
 
 console.log();
 console.log('==========================================');
-console.log(' Cloudron will use the following settings ');
-console.log('==========================================');
-console.log();
-console.log(' Environment:                    ', config.CLOUDRON ? 'CLOUDRON' : 'TEST');
-console.log(' Version:                        ', config.version());
-console.log(' Admin Origin:                   ', config.adminOrigin());
-console.log(' Appstore API server origin:     ', config.apiServerOrigin());
-console.log(' Appstore Web server origin:     ', config.webServerOrigin());
-console.log();
+console.log(`           Cloudron ${constants.VERSION}  `);
 console.log('==========================================');
 console.log();
 
 async.series([
     server.start,
     ldap.start,
-    simpleauth.start,
-    appHealthMonitor.start,
-    oauthproxy.start
+    dockerProxy.start
 ], function (error) {
     if (error) {
         console.error('Error starting server', error);
         process.exit(1);
     }
+    console.log('Cloudron is up and running');
 });
 
 var NOOP_CALLBACK = function () { };
 
 process.on('SIGINT', function () {
+    console.log('Received SIGINT. Shutting down.');
+
     server.stop(NOOP_CALLBACK);
     ldap.stop(NOOP_CALLBACK);
-    simpleauth.stop(NOOP_CALLBACK);
-    oauthproxy.stop(NOOP_CALLBACK);
+    dockerProxy.stop(NOOP_CALLBACK);
     setTimeout(process.exit.bind(process), 3000);
 });
 
 process.on('SIGTERM', function () {
+    console.log('Received SIGTERM. Shutting down.');
+
     server.stop(NOOP_CALLBACK);
     ldap.stop(NOOP_CALLBACK);
-    simpleauth.stop(NOOP_CALLBACK);
-    oauthproxy.stop(NOOP_CALLBACK);
+    dockerProxy.stop(NOOP_CALLBACK);
     setTimeout(process.exit.bind(process), 3000);
 });

crashnotifier.js

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-var assert = require('assert'),
-    mailer = require('./src/mailer.js'),
-    safe = require('safetydance'),
-    path = require('path'),
-    util = require('util');
-
-var COLLECT_LOGS_CMD = path.join(__dirname, 'src/scripts/collectlogs.sh');
-
-function collectLogs(program, callback) {
-    assert.strictEqual(typeof program, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
-    callback(null, logs);
-}
-
-function sendCrashNotification(processName) {
-    collectLogs(processName, function (error, result) {
-        if (error) {
-            console.error('Failed to collect logs.', error);
-            result = util.format('Failed to collect logs.', error);
-        }
-
-        console.log('Sending crash notification email for', processName);
-        mailer.sendCrashNotification(processName, result);
-    });
-}
-
-function main() {
-    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
-
-    var processName = process.argv[2];
-    console.log('Started crash notifier for', processName);
-
-    sendCrashNotification(processName);
-}
-
-main();

crashnotifierservice.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+
+'use strict';
+
+var database = require('./src/database.js');
+
+var crashNotifier = require('./src/crashnotifier.js');
+
+// This is triggered by systemd with the crashed unit name as argument
+function main() {
+    if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <unitName>');
+
+    var unitName = process.argv[2];
+    console.log('Started crash notifier for', unitName);
+
+    // eventlog api needs the db
+    database.initialize(function (error) {
+        if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
+
+        crashNotifier.sendFailureLogs(unitName, function (error) {
+            if (error) console.error(error);
+
+            process.exit();
+        });
+    });
+}
+
+main();

docs/makeDocs

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-set -eu
-
-./node_modules/.bin/apidoc -i src/routes -o docs

gulpfile.js

@@ -1,156 +0,0 @@
-/* jslint node:true */
-
-'use strict';
-
-var ejs = require('gulp-ejs'),
-    gulp = require('gulp'),
-    del = require('del'),
-    concat = require('gulp-concat'),
-    uglify = require('gulp-uglify'),
-    serve = require('gulp-serve'),
-    sass = require('gulp-sass'),
-    sourcemaps = require('gulp-sourcemaps'),
-    cssnano = require('gulp-cssnano'),
-    autoprefixer = require('gulp-autoprefixer'),
-    argv = require('yargs').argv;
-
-gulp.task('3rdparty', function () {
-    gulp.src([
-        'webadmin/src/3rdparty/**/*.js',
-        'webadmin/src/3rdparty/**/*.map',
-        'webadmin/src/3rdparty/**/*.css',
-        'webadmin/src/3rdparty/**/*.otf',
-        'webadmin/src/3rdparty/**/*.eot',
-        'webadmin/src/3rdparty/**/*.svg',
-        'webadmin/src/3rdparty/**/*.gif',
-        'webadmin/src/3rdparty/**/*.ttf',
-        'webadmin/src/3rdparty/**/*.woff',
-        'webadmin/src/3rdparty/**/*.woff2'
-        ])
-        .pipe(gulp.dest('webadmin/dist/3rdparty/'))
-        .pipe(gulp.dest('setup/splash/website/3rdparty'));
-
-    gulp.src('node_modules/bootstrap-sass/assets/javascripts/bootstrap.min.js')
-        .pipe(gulp.dest('webadmin/dist/3rdparty/js'))
-        .pipe(gulp.dest('setup/splash/website/3rdparty/js'));
-});
-
-
-// --------------
-// JavaScript
-// --------------
-
-gulp.task('js', ['js-index', 'js-setup', 'js-update'], function () {});
-
-var oauth = {
-    clientId: argv.clientId || 'cid-webadmin',
-    clientSecret: argv.clientSecret || 'unused',
-    apiOrigin: argv.apiOrigin || ''
-};
-
-console.log();
-console.log('Using OAuth credentials:');
-console.log(' ClientId:     %s', oauth.clientId);
-console.log(' ClientSecret: %s', oauth.clientSecret);
-console.log(' Cloudron API: %s', oauth.apiOrigin || 'default');
-console.log();
-
-gulp.task('js-index', function () {
-    gulp.src([
-        'webadmin/src/js/index.js',
-        'webadmin/src/js/client.js',
-        'webadmin/src/js/appstore.js',
-        'webadmin/src/js/main.js',
-        'webadmin/src/views/*.js'
-        ])
-        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
-        .pipe(sourcemaps.init())
-        .pipe(concat('index.js', { newLine: ';' }))
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'));
-});
-
-gulp.task('js-setup', function () {
-    gulp.src(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'])
-        .pipe(ejs({ oauth: oauth }, { ext: '.js' }))
-        .pipe(sourcemaps.init())
-        .pipe(concat('setup.js', { newLine: ';' }))
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'));
-});
-
-gulp.task('js-update', function () {
-    gulp.src(['webadmin/src/js/update.js'])
-        .pipe(sourcemaps.init())
-        .pipe(uglify())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist/js'))
-        .pipe(gulp.dest('setup/splash/website/js'));
-});
-
-
-// --------------
-// HTML
-// --------------
-
-gulp.task('html', ['html-views', 'html-update', 'html-templates'], function () {
-    return gulp.src('webadmin/src/*.html').pipe(gulp.dest('webadmin/dist'));
-});
-
-gulp.task('html-update', function () {
-    return gulp.src(['webadmin/src/update.html']).pipe(gulp.dest('setup/splash/website'));
-});
-
-gulp.task('html-views', function () {
-    return gulp.src('webadmin/src/views/**/*.html').pipe(gulp.dest('webadmin/dist/views'));
-});
-
-gulp.task('html-templates', function () {
-    return gulp.src('webadmin/src/templates/**/*.html').pipe(gulp.dest('webadmin/dist/templates'));
-});
-
-// --------------
-// CSS
-// --------------
-
-gulp.task('css', function () {
-    return gulp.src('webadmin/src/*.scss')
-        .pipe(sourcemaps.init())
-        .pipe(sass({ includePaths: ['node_modules/bootstrap-sass/assets/stylesheets/'] }).on('error', sass.logError))
-        .pipe(autoprefixer())
-        .pipe(cssnano())
-        .pipe(sourcemaps.write())
-        .pipe(gulp.dest('webadmin/dist'))
-        .pipe(gulp.dest('setup/splash/website'));
-});
-
-gulp.task('images', function () {
-    return gulp.src('webadmin/src/img/**')
-        .pipe(gulp.dest('webadmin/dist/img'));
-});
-
-// --------------
-// Utilities
-// --------------
-
-gulp.task('watch', ['default'], function () {
-    gulp.watch(['webadmin/src/*.scss'], ['css']);
-    gulp.watch(['webadmin/src/img/*'], ['images']);
-    gulp.watch(['webadmin/src/**/*.html'], ['html']);
-    gulp.watch(['webadmin/src/views/*.html'], ['html-views']);
-    gulp.watch(['webadmin/src/templates/*.html'], ['html-templates']);
-    gulp.watch(['webadmin/src/js/update.js'], ['js-update']);
-    gulp.watch(['webadmin/src/js/setup.js', 'webadmin/src/js/client.js'], ['js-setup']);
-    gulp.watch(['webadmin/src/js/index.js', 'webadmin/src/js/client.js', 'webadmin/src/js/appstore.js', 'webadmin/src/js/main.js', 'webadmin/src/views/*.js'], ['js-index']);
-    gulp.watch(['webadmin/src/3rdparty/**/*'], ['3rdparty']);
-});
-
-gulp.task('clean', function () {
-    del.sync(['webadmin/dist', 'setup/splash/website']);
-});
-
-gulp.task('default', ['clean', 'html', 'js', '3rdparty', 'images', 'css'], function () {});
-
-gulp.task('develop', ['watch'], serve({ root: 'webadmin/dist', port: 4000 }));

installer.sh.ejs

@@ -1,164 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-echo ""
-echo "======== Cloudron Installer ========"
-echo ""
-
-if [ $# -lt 4 ]; then
-    echo "Usage: ./installer.sh <fqdn> <aws key id> <aws key secret> <bucket> <provider> <revision>"
-    exit 1
-fi
-
-# commandline arguments
-readonly fqdn="${1}"
-readonly aws_access_key_id="${2}"
-readonly aws_access_key_secret="${3}"
-readonly aws_backup_bucket="${4}"
-readonly provider="${5}"
-readonly revision="${6}"
-
-# environment specific urls
-<% if (env === 'prod') { %>
-readonly api_server_origin="https://api.cloudron.io"
-readonly web_server_origin="https://cloudron.io"
-<% } else { %>
-readonly api_server_origin="https://api.<%= env %>.cloudron.io"
-readonly web_server_origin="https://<%= env %>.cloudron.io"
-<% } %>
-readonly release_bucket_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases"
-readonly versions_url="https://s3.amazonaws.com/<%= env %>-cloudron-releases/versions.json"
-readonly installer_code_url="${release_bucket_url}/box-${revision}.tar.gz"
-
-# runtime consts
-readonly installer_code_file="/tmp/box.tar.gz"
-readonly installer_tmp_dir="/tmp/box"
-readonly cert_folder="/tmp/certificates"
-
-# check for fqdn in /ets/hosts
-echo "[INFO] checking for hostname entry"
-readonly hostentry_found=$(grep "${fqdn}" /etc/hosts || true)
-if [[ -z $hostentry_found ]]; then
-    echo "[WARNING] No entry for ${fqdn} found in /etc/hosts"
-    echo "Adding an entry ..."
-
-    cat >> /etc/hosts <<EOF
-
-# The following line was added by the Cloudron installer script
-127.0.1.1 ${fqdn} ${fqdn}
-EOF
-else
-    echo "Valid hostname entry found in /etc/hosts"
-fi
-echo ""
-
-echo "[INFO] ensure minimal dependencies ..."
-apt-get update
-apt-get install -y curl
-echo ""
-
-echo "[INFO] Generating certificates ..."
-rm -rf "${cert_folder}"
-mkdir -p "${cert_folder}"
-
-cat > "${cert_folder}/CONFIG" <<EOF
-[ req ]
-default_bits           = 1024
-default_keyfile        = keyfile.pem
-distinguished_name     = req_distinguished_name
-prompt                 = no
-req_extensions         = v3_req
-
-[ req_distinguished_name ]
-C                      = DE
-ST                     = Berlin
-L                      = Berlin
-O                      = Cloudron UG
-OU                     = Cloudron
-CN                     = ${fqdn}
-emailAddress           = cert@cloudron.io
-
-[ v3_req ]
-# Extensions to add to a certificate request
-basicConstraints = CA:FALSE
-keyUsage = nonRepudiation, digitalSignature, keyEncipherment
-subjectAltName = @alt_names
-
-[alt_names]
-DNS.1 = ${fqdn}
-DNS.2 = *.${fqdn}
-EOF
-
-# generate cert files
-openssl genrsa 2048 > "${cert_folder}/host.key"
-openssl req -new -out "${cert_folder}/host.csr" -key "${cert_folder}/host.key" -config "${cert_folder}/CONFIG"
-openssl x509 -req -days 3650 -in "${cert_folder}/host.csr" -signkey "${cert_folder}/host.key" -out "${cert_folder}/host.cert" -extensions v3_req -extfile "${cert_folder}/CONFIG"
-
-# make them json compatible, by collapsing to one line
-tls_cert=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.cert")
-tls_key=$(sed ':a;N;$!ba;s/\n/\\n/g' "${cert_folder}/host.key")
-echo ""
-
-echo "[INFO] Fetching installer code ..."
-curl "${installer_code_url}" -o "${installer_code_file}"
-echo ""
-
-echo "[INFO] Extracting installer code to ${installer_tmp_dir} ..."
-rm -rf "${installer_tmp_dir}" && mkdir -p "${installer_tmp_dir}"
-tar xvf "${installer_code_file}" -C "${installer_tmp_dir}"
-echo ""
-
-echo "Creating initial provisioning config ..."
-cat > /root/provision.json <<EOF
-{
-    "sourceTarballUrl": "",
-    "data": {
-        "apiServerOrigin": "${api_server_origin}",
-        "webServerOrigin": "${web_server_origin}",
-        "fqdn": "${fqdn}",
-        "token": "",
-        "isCustomDomain": true,
-        "boxVersionsUrl": "${versions_url}",
-        "version": "",
-        "tlsCert": "${tls_cert}",
-        "tlsKey": "${tls_key}",
-        "provider": "${provider}",
-        "backupConfig": {
-            "provider": "s3",
-            "accessKeyId": "${aws_access_key_id}",
-            "secretAccessKey": "${aws_access_key_secret}",
-            "bucket": "${aws_backup_bucket}",
-            "prefix": "backups"
-        },
-        "dnsConfig": {
-            "provider": "route53",
-            "accessKeyId": "${aws_access_key_id}",
-            "secretAccessKey": "${aws_access_key_secret}"
-        },
-        "tlsConfig": {
-            "provider": "letsencrypt-<%= env %>"
-        }
-    }
-}
-EOF
-
-echo "[INFO] Running Ubuntu initializing script ..."
-/bin/bash "${installer_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${revision}" selfhosting
-echo ""
-
-echo "[INFO] Reloading systemd daemon ..."
-systemctl daemon-reload
-echo ""
-
-echo "[INFO] Restart docker ..."
-systemctl restart docker
-echo ""
-
-echo "[FINISHED] Now starting Cloudron init jobs ..."
-systemctl start box-setup
-
-# TODO this is only for convenience we should probably just let the user do a restart
-sleep 5 && sync
-systemctl start cloudron-installer
-journalctl -u cloudron-installer.service -f

installer/npm-shrinkwrap.json

@@ -1,516 +0,0 @@
-{
-  "name": "installer",
-  "version": "0.0.1",
-  "dependencies": {
-    "async": {
-      "version": "1.5.0",
-      "from": "async@>=1.5.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz"
-    },
-    "body-parser": {
-      "version": "1.14.1",
-      "from": "body-parser@>=1.12.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz",
-      "dependencies": {
-        "bytes": {
-          "version": "2.1.0",
-          "from": "bytes@2.1.0",
-          "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz"
-        },
-        "content-type": {
-          "version": "1.0.1",
-          "from": "content-type@>=1.0.1 <1.1.0",
-          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
-        },
-        "depd": {
-          "version": "1.1.0",
-          "from": "depd@>=1.1.0 <1.2.0",
-          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
-        },
-        "http-errors": {
-          "version": "1.3.1",
-          "from": "http-errors@>=1.3.1 <1.4.0",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-          "dependencies": {
-            "inherits": {
-              "version": "2.0.1",
-              "from": "inherits@>=2.0.1 <2.1.0",
-              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-            },
-            "statuses": {
-              "version": "1.2.1",
-              "from": "statuses@>=1.0.0 <2.0.0",
-              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
-            }
-          }
-        },
-        "iconv-lite": {
-          "version": "0.4.12",
-          "from": "iconv-lite@0.4.12",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz"
-        },
-        "on-finished": {
-          "version": "2.3.0",
-          "from": "on-finished@>=2.3.0 <2.4.0",
-          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "dependencies": {
-            "ee-first": {
-              "version": "1.1.1",
-              "from": "ee-first@1.1.1",
-              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-            }
-          }
-        },
-        "qs": {
-          "version": "5.1.0",
-          "from": "qs@5.1.0",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz"
-        },
-        "raw-body": {
-          "version": "2.1.4",
-          "from": "raw-body@>=2.1.4 <2.2.0",
-          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz",
-          "dependencies": {
-            "unpipe": {
-              "version": "1.0.0",
-              "from": "unpipe@1.0.0",
-              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
-            }
-          }
-        },
-        "type-is": {
-          "version": "1.6.9",
-          "from": "type-is@>=1.6.9 <1.7.0",
-          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
-          "dependencies": {
-            "media-typer": {
-              "version": "0.3.0",
-              "from": "media-typer@0.3.0",
-              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
-            },
-            "mime-types": {
-              "version": "2.1.7",
-              "from": "mime-types@>=2.1.7 <2.2.0",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "dependencies": {
-                "mime-db": {
-                  "version": "1.19.0",
-                  "from": "mime-db@>=1.19.0 <1.20.0",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "connect-lastmile": {
-      "version": "0.0.13",
-      "from": "connect-lastmile@0.0.13",
-      "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz",
-      "dependencies": {
-        "debug": {
-          "version": "2.1.3",
-          "from": "debug@>=2.1.0 <2.2.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz",
-          "dependencies": {
-            "ms": {
-              "version": "0.7.0",
-              "from": "ms@0.7.0",
-              "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz"
-            }
-          }
-        }
-      }
-    },
-    "debug": {
-      "version": "2.2.0",
-      "from": "debug@>=2.1.1 <3.0.0",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
-      "dependencies": {
-        "ms": {
-          "version": "0.7.1",
-          "from": "ms@0.7.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-        }
-      }
-    },
-    "express": {
-      "version": "4.13.3",
-      "from": "express@>=4.11.2 <5.0.0",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz",
-      "dependencies": {
-        "accepts": {
-          "version": "1.2.13",
-          "from": "accepts@>=1.2.12 <1.3.0",
-          "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
-          "dependencies": {
-            "mime-types": {
-              "version": "2.1.7",
-              "from": "mime-types@>=2.1.6 <2.2.0",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "dependencies": {
-                "mime-db": {
-                  "version": "1.19.0",
-                  "from": "mime-db@>=1.19.0 <1.20.0",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
-                }
-              }
-            },
-            "negotiator": {
-              "version": "0.5.3",
-              "from": "negotiator@0.5.3",
-              "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
-            }
-          }
-        },
-        "array-flatten": {
-          "version": "1.1.1",
-          "from": "array-flatten@1.1.1",
-          "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
-        },
-        "content-disposition": {
-          "version": "0.5.0",
-          "from": "content-disposition@0.5.0",
-          "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz"
-        },
-        "content-type": {
-          "version": "1.0.1",
-          "from": "content-type@>=1.0.1 <1.1.0",
-          "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz"
-        },
-        "cookie": {
-          "version": "0.1.3",
-          "from": "cookie@0.1.3",
-          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz"
-        },
-        "cookie-signature": {
-          "version": "1.0.6",
-          "from": "cookie-signature@1.0.6",
-          "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
-        },
-        "depd": {
-          "version": "1.0.1",
-          "from": "depd@>=1.0.1 <1.1.0",
-          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
-        },
-        "escape-html": {
-          "version": "1.0.2",
-          "from": "escape-html@1.0.2",
-          "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz"
-        },
-        "etag": {
-          "version": "1.7.0",
-          "from": "etag@>=1.7.0 <1.8.0",
-          "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
-        },
-        "finalhandler": {
-          "version": "0.4.0",
-          "from": "finalhandler@0.4.0",
-          "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz",
-          "dependencies": {
-            "unpipe": {
-              "version": "1.0.0",
-              "from": "unpipe@>=1.0.0 <1.1.0",
-              "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
-            }
-          }
-        },
-        "fresh": {
-          "version": "0.3.0",
-          "from": "fresh@0.3.0",
-          "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
-        },
-        "merge-descriptors": {
-          "version": "1.0.0",
-          "from": "merge-descriptors@1.0.0",
-          "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
-        },
-        "methods": {
-          "version": "1.1.1",
-          "from": "methods@>=1.1.1 <1.2.0",
-          "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
-        },
-        "on-finished": {
-          "version": "2.3.0",
-          "from": "on-finished@>=2.3.0 <2.4.0",
-          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "dependencies": {
-            "ee-first": {
-              "version": "1.1.1",
-              "from": "ee-first@1.1.1",
-              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-            }
-          }
-        },
-        "parseurl": {
-          "version": "1.3.0",
-          "from": "parseurl@>=1.3.0 <1.4.0",
-          "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
-        },
-        "path-to-regexp": {
-          "version": "0.1.7",
-          "from": "path-to-regexp@0.1.7",
-          "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
-        },
-        "proxy-addr": {
-          "version": "1.0.8",
-          "from": "proxy-addr@>=1.0.8 <1.1.0",
-          "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz",
-          "dependencies": {
-            "forwarded": {
-              "version": "0.1.0",
-              "from": "forwarded@>=0.1.0 <0.2.0",
-              "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz"
-            },
-            "ipaddr.js": {
-              "version": "1.0.1",
-              "from": "ipaddr.js@1.0.1",
-              "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz"
-            }
-          }
-        },
-        "qs": {
-          "version": "4.0.0",
-          "from": "qs@4.0.0",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
-        },
-        "range-parser": {
-          "version": "1.0.3",
-          "from": "range-parser@>=1.0.2 <1.1.0",
-          "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
-        },
-        "send": {
-          "version": "0.13.0",
-          "from": "send@0.13.0",
-          "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz",
-          "dependencies": {
-            "destroy": {
-              "version": "1.0.3",
-              "from": "destroy@1.0.3",
-              "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz"
-            },
-            "http-errors": {
-              "version": "1.3.1",
-              "from": "http-errors@>=1.3.1 <1.4.0",
-              "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
-              "dependencies": {
-                "inherits": {
-                  "version": "2.0.1",
-                  "from": "inherits@>=2.0.1 <2.1.0",
-                  "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-                }
-              }
-            },
-            "mime": {
-              "version": "1.3.4",
-              "from": "mime@1.3.4",
-              "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
-            },
-            "ms": {
-              "version": "0.7.1",
-              "from": "ms@0.7.1",
-              "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
-            },
-            "statuses": {
-              "version": "1.2.1",
-              "from": "statuses@>=1.2.1 <1.3.0",
-              "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
-            }
-          }
-        },
-        "serve-static": {
-          "version": "1.10.0",
-          "from": "serve-static@>=1.10.0 <1.11.0",
-          "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
-        },
-        "type-is": {
-          "version": "1.6.9",
-          "from": "type-is@>=1.6.9 <1.7.0",
-          "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz",
-          "dependencies": {
-            "media-typer": {
-              "version": "0.3.0",
-              "from": "media-typer@0.3.0",
-              "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
-            },
-            "mime-types": {
-              "version": "2.1.7",
-              "from": "mime-types@>=2.1.6 <2.2.0",
-              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz",
-              "dependencies": {
-                "mime-db": {
-                  "version": "1.19.0",
-                  "from": "mime-db@>=1.19.0 <1.20.0",
-                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz"
-                }
-              }
-            }
-          }
-        },
-        "utils-merge": {
-          "version": "1.0.0",
-          "from": "utils-merge@1.0.0",
-          "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
-        },
-        "vary": {
-          "version": "1.0.1",
-          "from": "vary@>=1.0.1 <1.1.0",
-          "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
-        }
-      }
-    },
-    "json": {
-      "version": "9.0.3",
-      "from": "json@>=9.0.3 <10.0.0",
-      "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz"
-    },
-    "morgan": {
-      "version": "1.6.1",
-      "from": "morgan@>=1.5.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz",
-      "dependencies": {
-        "basic-auth": {
-          "version": "1.0.3",
-          "from": "basic-auth@>=1.0.3 <1.1.0",
-          "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz"
-        },
-        "depd": {
-          "version": "1.0.1",
-          "from": "depd@>=1.0.1 <1.1.0",
-          "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz"
-        },
-        "on-finished": {
-          "version": "2.3.0",
-          "from": "on-finished@>=2.3.0 <2.4.0",
-          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-          "dependencies": {
-            "ee-first": {
-              "version": "1.1.1",
-              "from": "ee-first@1.1.1",
-              "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
-            }
-          }
-        },
-        "on-headers": {
-          "version": "1.0.1",
-          "from": "on-headers@>=1.0.0 <1.1.0",
-          "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz"
-        }
-      }
-    },
-    "proxy-middleware": {
-      "version": "0.15.0",
-      "from": "proxy-middleware@>=0.15.0 <0.16.0",
-      "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz"
-    },
-    "safetydance": {
-      "version": "0.0.19",
-      "from": "safetydance@0.0.19",
-      "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz"
-    },
-    "semver": {
-      "version": "5.1.0",
-      "from": "semver@>=5.1.0 <6.0.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz"
-    },
-    "superagent": {
-      "version": "0.21.0",
-      "from": "superagent@>=0.21.0 <0.22.0",
-      "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz",
-      "dependencies": {
-        "component-emitter": {
-          "version": "1.1.2",
-          "from": "component-emitter@1.1.2",
-          "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz"
-        },
-        "cookiejar": {
-          "version": "2.0.1",
-          "from": "cookiejar@2.0.1",
-          "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz"
-        },
-        "extend": {
-          "version": "1.2.1",
-          "from": "extend@>=1.2.1 <1.3.0",
-          "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz"
-        },
-        "form-data": {
-          "version": "0.1.3",
-          "from": "form-data@0.1.3",
-          "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz",
-          "dependencies": {
-            "async": {
-              "version": "0.9.2",
-              "from": "async@>=0.9.0 <0.10.0",
-              "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz"
-            },
-            "combined-stream": {
-              "version": "0.0.7",
-              "from": "combined-stream@>=0.0.4 <0.1.0",
-              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
-              "dependencies": {
-                "delayed-stream": {
-                  "version": "0.0.5",
-                  "from": "delayed-stream@0.0.5",
-                  "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz"
-                }
-              }
-            }
-          }
-        },
-        "formidable": {
-          "version": "1.0.14",
-          "from": "formidable@1.0.14",
-          "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz"
-        },
-        "methods": {
-          "version": "1.0.1",
-          "from": "methods@1.0.1",
-          "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz"
-        },
-        "mime": {
-          "version": "1.2.11",
-          "from": "mime@1.2.11",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz"
-        },
-        "qs": {
-          "version": "1.2.0",
-          "from": "qs@1.2.0",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz"
-        },
-        "readable-stream": {
-          "version": "1.0.27-1",
-          "from": "readable-stream@1.0.27-1",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz",
-          "dependencies": {
-            "core-util-is": {
-              "version": "1.0.1",
-              "from": "core-util-is@>=1.0.0 <1.1.0",
-              "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz"
-            },
-            "inherits": {
-              "version": "2.0.1",
-              "from": "inherits@>=2.0.1 <2.1.0",
-              "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
-            },
-            "isarray": {
-              "version": "0.0.1",
-              "from": "isarray@0.0.1",
-              "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
-            },
-            "string_decoder": {
-              "version": "0.10.31",
-              "from": "string_decoder@>=0.10.0 <0.11.0",
-              "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
-            }
-          }
-        },
-        "reduce-component": {
-          "version": "1.0.1",
-          "from": "reduce-component@1.0.1",
-          "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz"
-        }
-      }
-    }
-  }
-}

installer/package.json

@@ -1,47 +0,0 @@
-{
-  "name": "installer",
-  "description": "Cloudron Installer",
-  "version": "0.0.1",
-  "private": "true",
-  "author": {
-    "name": "Cloudron authors"
-  },
-  "repository": {
-    "type": "git"
-  },
-  "engines": [
-    "node >=4.0.0 <=4.1.1"
-  ],
-  "dependencies": {
-    "async": "^1.5.0",
-    "body-parser": "^1.12.0",
-    "connect-lastmile": "0.0.13",
-    "debug": "^2.1.1",
-    "express": "^4.11.2",
-    "json": "^9.0.3",
-    "morgan": "^1.5.1",
-    "proxy-middleware": "^0.15.0",
-    "safetydance": "0.0.19",
-    "semver": "^5.1.0",
-    "superagent": "^0.21.0"
-  },
-  "devDependencies": {
-    "colors": "^1.1.2",
-    "commander": "^2.8.1",
-    "expect.js": "^0.3.1",
-    "istanbul": "^0.3.5",
-    "lodash": "^3.2.0",
-    "mocha": "^2.1.0",
-    "nock": "^0.59.1",
-    "sleep": "^3.0.0",
-    "superagent-sync": "^0.2.0",
-    "supererror": "^0.7.0",
-    "yesno": "0.0.1"
-  },
-  "scripts": {
-    "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test",
-    "precommit": "/bin/true",
-    "prepush": "npm test",
-    "postmerge": "/bin/true"
-  }
-}

installer/src/installer.js

@@ -1,112 +0,0 @@
-/* jslint node: true */
-
-'use strict';
-
-var assert = require('assert'),
-    child_process = require('child_process'),
-    debug = require('debug')('installer:installer'),
-    path = require('path'),
-    safe = require('safetydance'),
-    semver = require('semver'),
-    superagent = require('superagent'),
-    util = require('util');
-
-exports = module.exports = {
-    InstallerError: InstallerError,
-
-    provision: provision,
-
-    _ensureVersion: ensureVersion
-};
-
-var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'),
-    SUDO = '/usr/bin/sudo';
-
-function InstallerError(reason, info) {
-    Error.call(this);
-    Error.captureStackTrace(this, this.constructor);
-
-    this.name = this.constructor.name;
-    this.reason = reason;
-    this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info);
-}
-util.inherits(InstallerError, Error);
-InstallerError.INTERNAL_ERROR = 1;
-InstallerError.ALREADY_PROVISIONED = 2;
-
-// system until file has KillMode=control-group to bring down child processes
-function spawn(tag, cmd, args, callback) {
-    assert.strictEqual(typeof tag, 'string');
-    assert.strictEqual(typeof cmd, 'string');
-    assert(util.isArray(args));
-    assert.strictEqual(typeof callback, 'function');
-
-    var cp = child_process.spawn(cmd, args, { timeout: 0 });
-    cp.stdout.setEncoding('utf8');
-    cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); });
-    cp.stderr.setEncoding('utf8');
-    cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); });
-
-    cp.on('error', function (error) {
-        debug('%s : child process errored %s', tag, error.message);
-        callback(error);
-    });
-
-    cp.on('exit', function (code, signal) {
-        debug('%s : child process exited. code: %d signal: %d', tag, code, signal);
-        if (signal) return callback(new Error('Exited with signal ' + signal));
-        if (code !== 0) return callback(new Error('Exited with code ' + code));
-
-        callback(null);
-    });
-}
-
-function ensureVersion(args, callback) {
-    assert.strictEqual(typeof args, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!args.data || !args.data.boxVersionsUrl) return callback(new Error('No boxVersionsUrl specified'));
-
-    if (args.sourceTarballUrl) return callback(null, args);
-
-    superagent.get(args.data.boxVersionsUrl).end(function (error, result) {
-        if (error && !error.response) return callback(error);
-        if (result.statusCode !== 200) return callback(new Error(util.format('Bad status: %s %s', result.statusCode, result.text)));
-
-        var versions = safe.JSON.parse(result.text);
-
-        if (!versions || typeof versions !== 'object') return callback(new Error('versions is not in valid format:' + safe.error));
-
-        var latestVersion = Object.keys(versions).sort(semver.compare).pop();
-        debug('ensureVersion: Latest version is %s etag:%s', latestVersion, result.header['etag']);
-
-        if (!versions[latestVersion]) return callback(new Error('No version available'));
-        if (!versions[latestVersion].sourceTarballUrl) return callback(new Error('No sourceTarballUrl specified'));
-
-        args.sourceTarballUrl = versions[latestVersion].sourceTarballUrl;
-        args.data.version = latestVersion;
-
-        callback(null, args);
-    });
-}
-
-function provision(args, callback) {
-    assert.strictEqual(typeof args, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (process.env.NODE_ENV === 'test') return callback(null);
-
-    ensureVersion(args, function (error, result) {
-        if (error) return callback(error);
-
-        var pargs = [ INSTALLER_CMD ];
-        pargs.push('--sourcetarballurl', result.sourceTarballUrl);
-        pargs.push('--data', JSON.stringify(result.data));
-
-        debug('provision: calling with args %j', pargs);
-
-        // sudo is required for update()
-        spawn('provision', SUDO, pargs, callback);
-    });
-}
-

installer/src/scripts/installer.sh

@@ -1,67 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly BOX_SRC_DIR=/home/yellowtent/box
-readonly DATA_DIR=/home/yellowtent/data
-
-readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly json="${script_dir}/../../node_modules/.bin/json"
-readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 300"
-
-readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no")
-
-# create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@
-(echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh
-chmod +x /home/yellowtent/provision.sh
-
-arg_source_tarball_url=""
-arg_data=""
-
-args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@")
-eval set -- "${args}"
-
-while true; do
-    case "$1" in
-    --sourcetarballurl) arg_source_tarball_url="$2";;
-    --data) arg_data="$2";;
-    --) break;;
-    *) echo "Unknown option $1"; exit 1;;
-    esac
-
-    shift 2
-done
-
-box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX)
-echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}"
-
-while true; do
-    if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi
-    echo "Failed to download source tarball, trying again"
-    sleep 5
-done
-while true; do
-    # for reasons unknown, the dtrace package will fail. but rebuilding second time will work
-    if cd "${box_src_tmp_dir}" && npm rebuild; then break; fi
-    echo "Failed to rebuild, trying again"
-    sleep 5
-done
-
-if [[ "${is_update}" == "yes" ]]; then
-    echo "Setting up update splash screen"
-    "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code
-    ${BOX_SRC_DIR}/setup/stop.sh # stop the old code
-fi
-
-# switch the codes
-rm -rf "${BOX_SRC_DIR}"
-mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}"
-chown -R yellowtent.yellowtent "${BOX_SRC_DIR}"
-
-# create a start file for testing. %q escapes args
-(echo -e "#!/bin/bash\n"; printf "%q " "${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}") > /home/yellowtent/setup_start.sh
-chmod +x /home/yellowtent/setup_start.sh
-
-echo "Calling box setup script"
-"${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}"
-

installer/src/server.js

@@ -1,144 +0,0 @@
-#!/usr/bin/env node
-
-/* jslint node: true */
-
-'use strict';
-
-var assert = require('assert'),
-    async = require('async'),
-    debug = require('debug')('installer:server'),
-    express = require('express'),
-    fs = require('fs'),
-    http = require('http'),
-    HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess,
-    installer = require('./installer.js'),
-    json = require('body-parser').json,
-    lastMile = require('connect-lastmile'),
-    morgan = require('morgan'),
-    superagent = require('superagent');
-
-exports = module.exports = {
-    start: start,
-    stop: stop
-};
-
-var PROVISION_CONFIG_FILE = '/root/provision.json';
-var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf';
-
-var gHttpServer = null; // update server; used for updates
-
-function provisionDigitalOcean(callback) {
-    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
-
-    superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) {
-        if (error || result.statusCode !== 200) {
-            console.error('Error getting metadata', error);
-            return callback(new Error('Error getting metadata'));
-        }
-
-        var userData = JSON.parse(result.body.user_data);
-
-        installer.provision(userData, callback);
-    });
-}
-
-function provisionLocal(callback) {
-    if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned
-
-    if (!fs.existsSync(PROVISION_CONFIG_FILE)) {
-        console.error('No provisioning data found at %s', PROVISION_CONFIG_FILE);
-        return callback(new Error('No provisioning data found'));
-    }
-
-    var userData = require(PROVISION_CONFIG_FILE);
-
-    installer.provision(userData, callback);
-}
-
-function update(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-
-    if (!req.body.sourceTarballUrl || typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided'));
-    if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided'));
-
-    debug('provision: received from box %j', req.body);
-
-    installer.provision(req.body, function (error) {
-        if (error) console.error(error);
-    });
-
-    next(new HttpSuccess(202, { }));
-}
-
-function startUpdateServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Starting update server');
-
-    var app = express();
-
-    var router = new express.Router();
-
-    if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false }));
-
-    app.use(json({ strict: true }))
-       .use(router)
-       .use(lastMile());
-
-    router.post('/api/v1/installer/update', update);
-
-    gHttpServer = http.createServer(app);
-    gHttpServer.on('error', console.error);
-
-    gHttpServer.listen(2020, '127.0.0.1', callback);
-}
-
-function stopUpdateServer(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    debug('Stopping update server');
-
-    if (!gHttpServer) return callback(null);
-
-    gHttpServer.close(callback);
-    gHttpServer = null;
-}
-
-function start(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var actions;
-
-    if (process.env.PROVISION === 'local') {
-        debug('Starting Installer in selfhost mode');
-
-        actions = [
-            startUpdateServer,
-            provisionLocal
-        ];
-    } else {    // current fallback, should be 'digitalocean' eventually, see initializeBaseUbuntuImage.sh
-        debug('Starting Installer in managed mode');
-
-        actions = [
-            startUpdateServer,
-            provisionDigitalOcean
-        ];
-    }
-
-    async.series(actions, callback);
-}
-
-function stop(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    async.series([
-        stopUpdateServer
-    ], callback);
-}
-
-if (require.main === module) {
-    start(function (error) {
-        if (error) console.error(error);
-    });
-}

installer/src/test/installer-test.js

@@ -1,179 +0,0 @@
-/* jslint node:true */
-/* global it:false */
-/* global describe:false */
-/* global before:false */
-/* global after:false */
-
-'use strict';
-
-var expect = require('expect.js'),
-    fs = require('fs'),
-    path = require('path'),
-    nock = require('nock'),
-    os = require('os'),
-    request = require('superagent'),
-    server = require('../server.js'),
-    installer = require('../installer.js'),
-    _ = require('lodash');
-
-var EXTERNAL_SERVER_URL = 'https://localhost:4443';
-var INTERNAL_SERVER_URL = 'http://localhost:2020';
-var APPSERVER_ORIGIN = 'http://appserver';
-var FQDN = os.hostname();
-
-describe('Server', function () {
-    this.timeout(5000);
-
-    before(function (done) {
-        var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string
-        var scope = nock('http://169.254.169.254')
-            .persist()
-            .get('/metadata/v1.json')
-            .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' });
-        done();
-    });
-
-    after(function (done) {
-        nock.cleanAll();
-        done();
-    });
-
-    describe('starts and stop', function () {
-        it('starts', function (done) {
-            server.start(done);
-        });
-
-        it('stops', function (done) {
-            server.stop(done);
-        });
-    });
-
-    describe('update (internal server)', function () {
-        before(function (done) {
-            server.start(done);
-        });
-        after(function (done) {
-            server.stop(done);
-        });
-
-        it('does not respond to provision', function (done) {
-            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(404);
-                done();
-            });
-        });
-
-        it('does not respond to restore', function (done) {
-            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(404);
-                done();
-            });
-        });
-
-        var data = {
-            sourceTarballUrl: "https://foo.tar.gz",
-
-            data: {
-                token: 'sometoken',
-                apiServerOrigin: APPSERVER_ORIGIN,
-                webServerOrigin: 'https://somethingelse.com',
-                fqdn: 'www.something.com',
-                tlsKey: 'key',
-                tlsCert: 'cert',
-                boxVersionsUrl: 'https://versions.json',
-                version: '0.1'
-            }
-        };
-
-        Object.keys(data).forEach(function (key) {
-            it('fails due to missing ' + key, function (done) {
-                var dataCopy = _.merge({ }, data);
-                delete dataCopy[key];
-
-                request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) {
-                    expect(error).to.not.be.ok();
-                    expect(result.statusCode).to.equal(400);
-                    done();
-                });
-            });
-        });
-
-        it('succeeds', function (done) {
-            request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) {
-                expect(error).to.not.be.ok();
-                expect(result.statusCode).to.equal(202);
-                done();
-            });
-        });
-    });
-
-    describe('ensureVersion', function () {
-        before(function () {
-            process.env.NODE_ENV = undefined;
-        });
-
-        after(function () {
-            process.env.NODE_ENV = 'test';
-        });
-
-        it ('fails without data', function (done) {
-            installer._ensureVersion({}, function (error) {
-                expect(error).to.be.an(Error);
-                done();
-            });
-        });
-
-        it ('fails without boxVersionsUrl', function (done) {
-            installer._ensureVersion({ data: {}}, function (error) {
-                expect(error).to.be.an(Error);
-                done();
-            });
-        });
-
-        it ('succeeds with sourceTarballUrl', function (done) {
-            var data = {
-                sourceTarballUrl: 'sometarballurl',
-                data: {
-                    boxVersionsUrl: 'http://foobar/versions.json'
-                }
-            };
-
-            installer._ensureVersion(data, function (error, result) {
-                expect(error).to.equal(null);
-                expect(result).to.eql(data);
-                done();
-            });
-        });
-
-        it ('succeeds without sourceTarballUrl', function (done) {
-            var versions = {
-                '0.1.0': {
-                    sourceTarballUrl: 'sometarballurl1'
-                },
-                '0.2.0': {
-                    sourceTarballUrl: 'sometarballurl2'
-                }
-            };
-
-            var scope = nock('http://foobar')
-                .get('/versions.json')
-                .reply(200, JSON.stringify(versions), { 'Content-Type': 'application/json' });
-
-            var data = {
-                data: {
-                    boxVersionsUrl: 'http://foobar/versions.json'
-                }
-            };
-
-            installer._ensureVersion(data, function (error, result) {
-                expect(error).to.equal(null);
-                expect(result.sourceTarballUrl).to.equal(versions['0.2.0'].sourceTarballUrl);
-                expect(result.data.boxVersionsUrl).to.equal(data.data.boxVersionsUrl);
-                done();
-            });
-        });
-    });
-});
-

installer/systemd/box-setup.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly USER_HOME="/home/yellowtent"
-readonly APPS_SWAP_FILE="/apps.swap"
-readonly USER_DATA_FILE="/root/user_data.img"
-readonly USER_DATA_DIR="/home/yellowtent/data"
-
-# detect device
-if [[ -b "/dev/vda1" ]]; then
-    disk_device="/dev/vda1"
-fi
-
-if [[ -b "/dev/xvda1" ]]; then
-    disk_device="/dev/xvda1"
-fi
-
-# all sizes are in mb
-readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }')
-readonly swap_size="${physical_memory}" # if you change this, fix enoughResourcesAvailable() in client.js
-readonly app_count=$((${physical_memory} / 200)) # estimated app count
-readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }')
-readonly disk_size=$((disk_size_gb * 1024))
-readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp
-readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1
-
-echo "Disk device: ${disk_device}"
-echo "Physical memory: ${physical_memory}"
-echo "Estimated app count: ${app_count}"
-echo "Disk size: ${disk_size}"
-
-# Allocate swap for general app usage
-if [[ ! -f "${APPS_SWAP_FILE}" ]]; then
-    echo "Creating Apps swap file of size ${swap_size}M"
-    fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}"
-    chmod 600 "${APPS_SWAP_FILE}"
-    mkswap "${APPS_SWAP_FILE}"
-    swapon "${APPS_SWAP_FILE}"
-    echo "${APPS_SWAP_FILE}  none  swap  sw  0 0" >> /etc/fstab
-else
-    echo "Apps Swap file already exists"
-fi
-
-echo "Resizing data volume"
-home_data_size=$((disk_size - system_size - swap_size - ext4_reserved))
-echo "Resizing up btrfs user data to size ${home_data_size}M"
-umount "${USER_DATA_DIR}" || true
-# Do not preallocate (non-sparse). Doing so overallocates for data too much in advance and causes problems when using many apps with smaller data
-# fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data
-truncate -s "${home_data_size}m" "${USER_DATA_FILE}" # this will shrink it if the file had existed. this is useful when running this script on a live system
-mount -t btrfs -o loop,nosuid "${USER_DATA_FILE}" ${USER_DATA_DIR}
-btrfs filesystem resize max "${USER_DATA_DIR}"
-

migrations/20141021192552-db-create.js

@@ -1,5 +1,5 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
+'use strict';
+
 var url = require('url');
 
 exports.up = function(db, callback) {

migrations/20141021192554-db-init.js

@@ -1,5 +1,4 @@
-var dbm = require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 var fs = require('fs'),
     async = require('async'),

migrations/20150303114527-users-add-resetToken.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE users ADD COLUMN resetToken VARCHAR(128) DEFAULT ""', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150303160528-tokens-expires-bigint.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('DELETE FROM tokens', [], function (error) {

migrations/20150303171203-authcodes-add-expiresAt.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE authcodes ADD COLUMN expiresAt BIGINT NOT NULL', function (error) {
@@ -13,4 +12,4 @@ exports.down = function(db, callback) {
         if (error) console.error(error);
         callback(error);
     });
-};
+};

migrations/20150308054950-appportbindings-add-environmentVariable.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE appPortBindings ADD COLUMN environmentVariable VARCHAR(128) NOT NULL', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150309023251-appportbindings-drop-containerport.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE appPortBindings DROP COLUMN containerPort', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150311120713-tokens-rename-userid-to-identifier.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('DELETE FROM tokens', [], function (error) {

migrations/20150323044254-apps-drop-version.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps DROP COLUMN version', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150323174906-apps-alter-health.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps DROP COLUMN healthy, ADD COLUMN health VARCHAR(128)', [], function (error) {

migrations/20150326072904-apps-add-lastBackupId.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps ADD COLUMN lastBackupId VARCHAR(128)', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150404093025-apps-add-createdAt.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps ADD COLUMN createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150430165335-settings-default-autoupdatepattern.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     // everyday at 1am
@@ -8,5 +7,4 @@ exports.up = function(db, callback) {
 
 exports.down = function(db, callback) {
     db.runSql('DELETE * FROM settings WHERE name="autoupdate_pattern"', [ ], callback);
-}
-
+};

migrations/20150430210225-settings-default-timezone.js

@@ -1,6 +1,6 @@
-dbm = dbm || require('db-migrate');
+'use strict';
+
 var safe = require('safetydance');
-var type = dbm.dataType;
 
 exports.up = function(db, callback) {
     var tz = safe.fs.readFileSync('/etc/timezone', 'utf8');
@@ -12,4 +12,3 @@ exports.up = function(db, callback) {
 exports.down = function(db, callback) {
     db.runSql('DELETE * FROM settings WHERE name="time_zone"', [ ], callback);
 };
-

migrations/20150615075901-users-add-unique-constraints.js

@@ -1,5 +1,5 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
+
 var async = require('async');
 
 exports.up = function(db, callback) {

migrations/20150615134751-users-adjust-username-and-email-constraints.js

@@ -1,5 +1,5 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
+
 var async = require('async');
 
 exports.up = function(db, callback) {

migrations/20150618192028-apps-add-lastManifestJson.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps ADD COLUMN lastManifestJson VARCHAR(2048)', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150710044740-apps-rename-lastBackupConfigJson.js

@@ -1,5 +1,4 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps CHANGE lastManifestJson lastBackupConfigJson VARCHAR(2048)', [], function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150710170847-apps-add-oldConfigJson.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps ADD COLUMN oldConfigJson VARCHAR(2048)', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20150719014338-settings-remove-all.js

@@ -1,5 +1,4 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('DELETE FROM settings', [ ], callback);

migrations/20151013073519-apps-add-oauthProxy.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20151015232915-clients-add-type.js

@@ -1,5 +1,5 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
+
 var async = require('async');
 
 exports.up = function(db, callback) {

migrations/20151016131005-apps-rename-accessRestriction.js

@@ -1,5 +1,4 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps CHANGE accessRestriction accessRestrictionJson VARCHAR(2048)', [], function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20151113091257-apps-alter-manifestJson.js

@@ -1,5 +1,4 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps MODIFY manifestJson TEXT', [], function (error) {

migrations/20151126111337-apps-alter-jsons.js

@@ -1,5 +1,5 @@
-dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
+
 var async = require('async');
 
 exports.up = function(db, callback) {

migrations/20160119113143-users-add-displayName.js

@@ -1,4 +1,4 @@
-dbm = dbm || require('db-migrate');
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE users ADD COLUMN displayName VARCHAR(512) DEFAULT ""', function (error) {

migrations/20160205135326-apps-add-memoryLimit.js

@@ -1,4 +1,4 @@
-dbm = dbm || require('db-migrate');
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE apps ADD COLUMN memoryLimit BIGINT DEFAULT 0', function (error) {

migrations/20160208031241-groups-add-table.js

@@ -1,8 +1,7 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
-    var cmd = "CREATE TABLE groups(" +
+    var cmd = "CREATE TABLE userGroups(" +
                 "id VARCHAR(128) NOT NULL UNIQUE," +
                 "name VARCHAR(128) NOT NULL UNIQUE," +
                 "PRIMARY KEY(id))";
@@ -14,7 +13,7 @@ exports.up = function(db, callback) {
 };
 
 exports.down = function(db, callback) {
-    db.runSql('DROP TABLE groups', function (error) {
+    db.runSql('DROP TABLE userGroups', function (error) {
         if (error) console.error(error);
         callback(error);
     });

migrations/20160208100000-groupMembers-add-table.js

@@ -1,11 +1,10 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
 	var cmd = "CREATE TABLE IF NOT EXISTS groupMembers(" +
     			"groupId VARCHAR(128) NOT NULL," +
     			"userId VARCHAR(128) NOT NULL," +
-    			"FOREIGN KEY(groupId) REFERENCES groups(id)," +
+    			"FOREIGN KEY(groupId) REFERENCES userGroups(id)," +
     			"FOREIGN KEY(userId) REFERENCES users(id));";
 
     db.runSql(cmd, function (error) {

migrations/20160208164735-groups-add-admin.js

@@ -1,14 +1,13 @@
 'use strict';
 
-var dbm = global.dbm || require('db-migrate');
 var async = require('async');
 
-var ADMIN_GROUP_ID = 'admin'; // see groups.js
+var ADMIN_GROUP_ID = 'admin'; // see constants.js
 
 exports.up = function(db, callback) {
 	async.series([
 		db.runSql.bind(db, 'START TRANSACTION;'),
-		db.runSql.bind(db, 'INSERT INTO groups (id, name) VALUES (?, ?)', [ ADMIN_GROUP_ID, 'admin' ]),
+		db.runSql.bind(db, 'INSERT INTO userGroups (id, name) VALUES (?, ?)', [ ADMIN_GROUP_ID, 'admin' ]),
 		function migrateAdminFlag(done) {
 			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
 				if (error) return done(error);

migrations/20160307172229-backups-add-table.js

@@ -1,5 +1,4 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     var cmd = "CREATE TABLE backups(" +

migrations/20160330215005-backups-add-configJson.js

@@ -1,5 +1,4 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE backups ADD COLUMN configJson TEXT', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20160404052453-backups-drop-configJson.js

@@ -1,5 +1,4 @@
-var dbm = dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE backups DROP COLUMN configJson', function (error) {
@@ -14,4 +13,3 @@ exports.down = function(db, callback) {
         callback(error);
     });
 };
-

migrations/20160404191651-backups-rename-filename-to-id.js

@@ -1,5 +1,4 @@
-var dbm = global.dbm || require('db-migrate');
-var type = dbm.dataType;
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE backups CHANGE filename id VARCHAR(128)', [], function (error) {

migrations/20160405083451-users-drop-username-null.js

@@ -1,4 +1,4 @@
-dbm = dbm || require('db-migrate');
+'use strict';
 
 exports.up = function(db, callback) {
     db.runSql('ALTER TABLE users MODIFY username VARCHAR(254) UNIQUE', [], function (error) {

migrations/20160419070047-apps-add-altDomain.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN altDomain VARCHAR(256)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN altDomain', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160430062446-eventlog-add-table.js

@@ -0,0 +1,23 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    var cmd = "CREATE TABLE eventlog(" +
+            "id VARCHAR(128) NOT NULL," +
+            "source TEXT," +
+            "creationTime TIMESTAMP," +
+            "action VARCHAR(128) NOT NULL," +
+            "data TEXT," +
+            "PRIMARY KEY (id))";
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE eventlog', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160506111925-users-add-showTutorial.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN showTutorial BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN showTutorial', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160527010012-mailboxes-add-table.js

@@ -0,0 +1,21 @@
+'use strict';
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160528090040-mailboxes-import-existing-users.js

@@ -0,0 +1,26 @@
+'use strict';
+
+var async = require('async');
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20160614021819-apps-drop-lastBackupConfigJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160621032914-apps-alter-installationProgress.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps MODIFY installationProgress VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160714130142-apps-add-xFrameOptions.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN xFrameOptions VARCHAR(512)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN xFrameOptions', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160831051352-settings-default-mailconfig.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT id FROM users', function (error, results) {
+        if (error) return callback(error);
+
+        // existing cloudrons have email enabled by default. future cloudrons will have it disabled by default
+        var enable = results.length !== 0;
+        db.runSql('INSERT settings (name, value) VALUES("mail_config", ?)', [ JSON.stringify({ enabled: enable }) ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="mail_config"', [ ], callback);
+};

migrations/20160921205726-mailboxes-add-ownerId.js

@@ -0,0 +1,73 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN ownerId VARCHAR(128)'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN ownerType VARCHAR(16)'),
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addGroupMailboxes(done) {
+            console.log('Importing group mailboxes');
+
+            db.all('SELECT id, name FROM userGroups', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (g, next) {
+                    db.runSql('INSERT INTO mailboxes (ownerId, ownerType, name) VALUES (?, ?, ?)', [ g.id, 'group', g.name ], function (error) {
+                        if (error) console.error('Error importing group ' + JSON.stringify(g) + error);
+                        next();
+                    });
+                }, done);
+            });
+        },
+        function addAppMailboxes(done) {
+            console.log('Importing app mail boxes');
+
+            db.all('SELECT id, location, manifestJson FROM apps', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (a, next) {
+                    var manifest = JSON.parse(a.manifestJson);
+                    if (!manifest.addons['sendmail'] && !manifest.addons['recvmail']) return next();
+
+                    var mailboxName = (a.location ? a.location : manifest.title.replace(/[^a-zA-Z0-9]/g, '')) + '.app';
+                    db.runSql('INSERT INTO mailboxes (ownerId, ownerType, name) VALUES (?, ?, ?)', [ a.id, 'app', mailboxName ], function (error) {
+                        if (error) console.error('Error importing app ' + JSON.stringify(a) + error);
+                        next();
+                    });
+                }, done);
+            });
+        },
+        function setUserMailboxOwnerIds(done) {
+            console.log('Setting owner id of user mailboxes and aliases');
+
+            db.all('SELECT id, username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (u,  next) {
+                    if (!u.username) return next();
+
+                    db.runSql('UPDATE mailboxes SET ownerId = ?, ownerType = ? WHERE name = ? OR aliasTarget = ?', [ u.id, 'user', u.username, u.username ], function (error) {
+                        if (error) console.error('Error setting ownerid ' + JSON.stringify(u) + error);
+                        next();
+                    });
+                }, done);
+            });
+        },
+        db.runSql.bind(db, 'COMMIT'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY ownerId VARCHAR(128) NOT NULL'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY ownerType VARCHAR(128) NOT NULL'),
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN ownerId', function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE mailboxes DROP COLUMN ownerType', function (error) {
+            if (error) console.error(error);
+            callback(error);
+        });
+    });
+};

migrations/20161111051640-appdb-add-sso.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN sso BOOLEAN DEFAULT 1', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN sso', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20161113014146-appdb-drop-oauthProxy.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN oauthProxy', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN oauthProxy BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20170117170621-users-drop-showTutorial.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE users DROP COLUMN showTutorial', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE users ADD COLUMN showTutorial BOOLEAN DEFAULT 0', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20170119234504-appdb-add-debugModeJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN debugModeJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN debugModeJson ', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20170223165502-backups-alter-dependsOn.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups MODIFY dependsOn TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups MODIFY dependsOn VARCHAR(4096)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20170325203037-appaddonconfigs-add-name.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs ADD COLUMN name VARCHAR(128)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE appAddonConfigs DROP COLUMN name', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20170414193114-database-utf8mb4.js

@@ -0,0 +1,14 @@
+'use strict';
+
+var url = require('url');
+
+exports.up = function(db, callback) {
+    var dbName = url.parse(process.env.DATABASE_URL).path.substr(1); // remove slash
+
+    // by default, mysql collates case insensitively. 'utf8_general_cs' is not available
+    db.runSql('ALTER DATABASE ' + dbName + '  DEFAULT CHARACTER SET=utf8mb4 DEFAULT COLLATE utf8mb4_unicode_ci', callback);
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20170418073650-backups-add-restoreConfigJson.js

@@ -0,0 +1,95 @@
+'use strict';
+
+var async = require('async');
+
+// from apps.js DO NOT UPDATE WHEN apps.js changes, as this is part of db migration!!
+function postProcess(result) {
+    try {
+        result.manifest = JSON.parse(result.manifestJson);
+        delete result.manifestJson;
+
+        result.oldConfig = JSON.parse(result.oldConfigJson);
+        delete result.oldConfigJson;
+
+        result.portBindings = { };
+        var hostPorts = result.hostPorts === null ? [ ] : result.hostPorts.split(',');
+        var environmentVariables = result.environmentVariables === null ? [ ] : result.environmentVariables.split(',');
+
+        delete result.hostPorts;
+        delete result.environmentVariables;
+
+        for (var i = 0; i < environmentVariables.length; i++) {
+            result.portBindings[environmentVariables[i]] = parseInt(hostPorts[i], 10);
+        }
+
+        result.accessRestriction = JSON.parse(result.accessRestrictionJson);
+        if (result.accessRestriction && !result.accessRestriction.users) result.accessRestriction.users = [];
+        delete result.accessRestrictionJson;
+
+        // TODO remove later once all apps have this attribute
+        result.xFrameOptions = result.xFrameOptions || 'SAMEORIGIN';
+
+        result.sso = !!result.sso; // make it bool
+
+        result.debugMode = JSON.parse(result.debugModeJson);
+        delete result.debugModeJson;
+    } catch (e) {
+        console.error('Failed to get restoreConfig for app.', e);
+        console.error('Falling back to empty values to make the update succeed.');
+        result.manifest = null;
+    }
+}
+
+// from apps.js DO NOT UPDATE WHEN apps.js changes, as this is part of db migration!!
+var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
+    'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain',
+    'apps.xFrameOptions', 'apps.sso', 'apps.debugModeJson' ].join(',');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups ADD COLUMN restoreConfigJson TEXT'),
+        // fill all the backups with restoreConfigs from current apps
+        function addRestoreConfigs(callback) {
+            console.log('Importing restoreConfigs');
+
+            var appQuery = 'SELECT ' + APPS_FIELDS_PREFIXED + ',' +
+                'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables' +
+                ' FROM apps LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId' +
+                ' GROUP BY apps.id ORDER BY apps.id';
+
+            db.all(appQuery, function (error, apps) {
+                if (error) return callback(error);
+
+                apps.forEach(postProcess);
+
+                async.eachSeries(apps, function (app, next) {
+                    if (app.manifest === null) return next();
+
+                    db.all('SELECT * FROM backups WHERE type="app" AND id LIKE "%app%\\_' + app.id + '\\_%"', function (error, backups) {
+                        if (error) return next(error);
+
+                        // from apps.js:getAppConfig()
+                        var restoreConfig = {
+                            manifest: app.manifest,
+                            location: app.location,
+                            accessRestriction: app.accessRestriction,
+                            portBindings: app.portBindings,
+                            memoryLimit: app.memoryLimit,
+                            xFrameOptions: app.xFrameOptions || 'SAMEORIGIN',
+                            altDomain: app.altDomain
+                        };
+
+                        async.eachSeries(backups, function (backup, next) {
+                            db.runSql('UPDATE backups SET restoreConfigJson=?,creationTime=creationTime WHERE id=?', [ JSON.stringify(restoreConfig), backup.id ], next);
+                        }, next);
+                    });
+                }, callback);
+            });
+        }
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN restoreConfigJson', callback);
+};

migrations/20170423032116-settings-set-default-retentionSecs.js

@@ -0,0 +1,22 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
+        if (error || results.length === 0) return callback(error);
+
+        var backupConfig = JSON.parse(results[0].value);
+        if (backupConfig.provider === 'filesystem') {
+            backupConfig.retentionSecs = 2 * 24 * 60 * 60; // 2 days
+        } else if (backupConfig.provider === 's3') { // S3
+            backupConfig.retentionSecs = -1;
+        } else if (backupConfig.provider === 'caas') {
+            backupConfig.retentionSecs = 10 * 24 * 60 * 60; // 10 days
+        }
+        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
+
+    });
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20170628173051-settings-default-mailrelay.js

@@ -0,0 +1,9 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('INSERT settings (name, value) VALUES("mail_relay", ?)', [ JSON.stringify({ provider: 'cloudron-smtp' }) ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DELETE * FROM settings WHERE name="mail_relay"', [ ], callback);
+};

migrations/20170714170908-appdb-add-robotsTxt.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN robotsTxt TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN robotsTxt', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20170731073447-eventlog-alter-data-and-source.js

@@ -0,0 +1,29 @@
+'use strict';
+
+// we used to have JSON as the db type for those two, however mariadb does not support it
+// and we never used any JSON related features, but have the TEXT pattern everywhere
+// This ensures all old cloudrons will have the columns altered
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE eventlog MODIFY data TEXT', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE eventlog MODIFY source TEXT', [], function (error) {
+            if (error) console.error(error);
+
+            callback(error);
+        });
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE eventlog MODIFY data TEXT', [], function (error) {
+        if (error) console.error(error);
+
+        db.runSql('ALTER TABLE eventlog MODIFY source TEXT', [], function (error) {
+            if (error) console.error(error);
+
+            callback(error);
+        });
+    });
+};

migrations/20170816211111-appdb-add-enableBackup.js.js

@@ -0,0 +1,16 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN enableBackup BOOLEAN DEFAULT 1', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN enableBackup', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+

migrations/20170911133441-settings-alter-value.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE settings MODIFY value TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE settings MODIFY value VARCHAR(512)', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20170919185109-settings-set-backupConfig-defaults.js

@@ -0,0 +1,25 @@
+'use strict';
+
+// ensure backupFolder and format are not empty
+exports.up = function(db, callback) {
+    db.all('SELECT * FROM settings WHERE name=?', [ 'backup_config' ], function (error, result) {
+        if (error || result.length === 0) return callback(error);
+
+        var value = JSON.parse(result[0].value);
+        value.format = 'tgz'; // set the format
+
+        if (value.provider === 'filesystem' && !value.backupFolder) {
+            value.backupFolder = '/var/backups'; // set the backupFolder
+        }
+
+        db.runSql('UPDATE settings SET value = ? WHERE name = ?', [ JSON.stringify(value), 'backup_config' ], function (error) {
+            if (error) console.error('Error setting ownerid ' + JSON.stringify(u) + error);
+            callback();
+        });
+
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

migrations/20170928003352-backups-add-format.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE backups ADD COLUMN format VARCHAR(16) DEFAULT "tgz"', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE backups DROP COLUMN format', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20171013003424-apps-add-newConfigJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN newConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN newConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20171116191443-backups-add-manifestJson.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups ADD COLUMN manifestJson TEXT'),
+
+        db.runSql.bind(db, 'START TRANSACTION;'),
+
+        // fill all the backups with restoreConfigs from current apps
+        function addManifests(callback) {
+            console.log('Importing manifests');
+
+            db.all('SELECT * FROM backups WHERE type="app"', function (error, backups) {
+                if (error) return callback(error);
+
+                async.eachSeries(backups, function (backup, next) {
+                    var m = backup.restoreConfigJson ? JSON.parse(backup.restoreConfigJson) : null;
+                    if (m) m = JSON.stringify(m.manifest);
+
+                    db.runSql('UPDATE backups SET manifestJson=? WHERE id=?', [ m, backup.id ], next);
+                }, callback);
+            });
+        },
+
+        db.runSql.bind(db, 'COMMIT'),
+
+        // remove the restoreConfig
+        db.runSql.bind(db, 'ALTER TABLE backups DROP COLUMN restoreConfigJson')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE backups DROP COLUMN manifestJson'),
+        db.runSql.bind(db, 'ALTER TABLE backups ADD COLUMN restoreConfigJson TEXT'),
+    ], callback);
+};
+

migrations/20171116203507-apps-rename-newConfigJson-to-updateConfigJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE newConfigJson updateConfigJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE updateConfigJson newConfigJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20171116224051-apps-rename-lastBackupId-to-restoreConfigJson.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE lastBackupId restoreConfigJson TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps CHANGE restoreConfigJson lastBackupId TEXT', [], function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20171118000000-ensure-collation-utf8_bin.js

@@ -0,0 +1,31 @@
+'use strict';
+
+// WARNING!!
+// At this point the default db collation is utf8mb4_unicode_ci however we already have foreign key constraits
+// already with tables on utf8_bin charset, so we cannot convert all tables here to utf8mb4 collation without
+// a reimport from a sql dump, as foreign keys across different collations are not supported
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'ALTER TABLE appPortBindings CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE apps CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE authcodes CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE backups CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE clients CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE eventlog CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE groupMembers CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE userGroups CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE migrations CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE settings CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE tokens CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin'),
+        db.runSql.bind(db, 'ALTER TABLE users CONVERT TO CHARACTER SET utf8 COLLATE utf8_bin')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    // nothing to be done here
+    callback();
+};

migrations/20171118000001-apps-add-domain.js

@@ -0,0 +1,70 @@
+'use strict';
+
+var async = require('async'),
+    safe = require('safetydance');
+
+exports.up = function(db, callback) {
+    // first check precondtion of domain entry in settings
+    db.all('SELECT * FROM settings WHERE name = ?', [ 'domain' ], function (error, result) {
+        if (error) return callback(error);
+
+        var domain = {};
+        if (result[0]) domain = safe.JSON.parse(result[0].value) || {};
+
+        async.series([
+            db.runSql.bind(db, 'START TRANSACTION;'),
+            function addAppsDomainColumn(done) {
+                db.runSql('ALTER TABLE apps ADD COLUMN domain VARCHAR(128)', [], done);
+            },
+            function setAppDomain(done) {
+                if (!domain.fqdn) return done(); // skip for new cloudrons without a domain
+                db.runSql('UPDATE apps SET domain = ?', [ domain.fqdn ], done);
+            },
+            function addAppsLocationDomainUniqueConstraint(done) {
+                db.runSql('ALTER TABLE apps ADD UNIQUE location_domain_unique_index (location, domain)', [], done);
+            },
+            function removePresetupAdminGroupIfNew(done) {
+                // do not delete on update, will update the record in setMailboxesDomain()
+                if (domain.fqdn) return done();
+
+                // this will be finally created once we have a domain when we create the owner in user.js
+                const ADMIN_GROUP_ID = 'admin'; // see constants.js
+                db.runSql('DELETE FROM userGroups WHERE id = ?', [ ADMIN_GROUP_ID ], function (error) {
+                    if (error) return done(error);
+
+                    db.runSql('DELETE FROM mailboxes WHERE ownerId = ?', [ ADMIN_GROUP_ID ], done);
+                });
+            },
+            function addMailboxesDomainColumn(done) {
+                db.runSql('ALTER TABLE mailboxes ADD COLUMN domain VARCHAR(128)', [], done);
+            },
+            function setMailboxesDomain(done) {
+                if (!domain.fqdn) return done(); // skip for new cloudrons without a domain
+                db.runSql('UPDATE mailboxes SET domain = ?', [ domain.fqdn ], done);
+            },
+            function dropAppsLocationUniqueConstraint(done) {
+                db.runSql('ALTER TABLE apps DROP INDEX location', [], done);
+            },
+            db.runSql.bind(db, 'COMMIT')
+        ], callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function dropMailboxesDomainColumn(done) {
+            db.runSql('ALTER TABLE mailboxes DROP COLUMN domain', [], done);
+        },
+        function dropLocationDomainUniqueConstraint(done) {
+            db.runSql('ALTER TABLE apps DROP INDEX location_domain_unique_index', [], done);
+        },
+        function dropAppsDomainColumn(done) {
+            db.runSql('ALTER TABLE apps DROP COLUMN domain', [], done);
+        },
+        function addAppsLocationUniqueConstraint(done) {
+            db.runSql('ALTER TABLE apps ADD UNIQUE location (location)', [], done);
+        },
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};

migrations/20171118000002-domains-add-table.js

@@ -0,0 +1,61 @@
+'use strict';
+
+var async = require('async'),
+    safe = require('safetydance'),
+    tld = require('tldjs');
+
+exports.up = function(db, callback) {
+    var fqdn, zoneName, configJson;
+
+    async.series([
+        function gatherDomain(done) {
+            db.all('SELECT * FROM settings WHERE name = ?', [ 'domain' ], function (error, result) {
+                if (error) return done(error);
+
+                var domain = {};
+                if (result[0]) domain = safe.JSON.parse(result[0].value) || {};
+
+                fqdn = domain.fqdn || ''; // will be null pre-setup
+                zoneName = domain.zoneName || tld.getDomain(fqdn) || fqdn;
+
+                done();
+            });
+        },
+        function gatherDNSConfig(done) {
+            db.all('SELECT * FROM settings WHERE name = ?', [ 'dns_config' ], function (error, result) {
+                if (error) return done(error);
+
+                configJson = (result[0] && result[0].value) ? result[0].value : JSON.stringify({ provider: 'manual'});
+
+                // caas dns config needs an fqdn
+                var config = JSON.parse(configJson);
+                if (config.provider === 'caas') config.fqdn = fqdn;
+                configJson = JSON.stringify(config);
+
+                done();
+            });
+        },
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function createDomainsTable(done) {
+            var cmd = `
+                CREATE TABLE domains(
+                domain VARCHAR(128) NOT NULL UNIQUE,
+                zoneName VARCHAR(128) NOT NULL,
+                configJson TEXT,
+                PRIMARY KEY (domain)) CHARACTER SET utf8 COLLATE utf8_bin
+            `;
+
+            db.runSql(cmd, [], done);
+        },
+        function addInitialDomain(done) {
+            if (!fqdn) return done();
+
+            db.runSql('INSERT INTO domains (domain, zoneName, configJson) VALUES (?, ?, ?)', [ fqdn, zoneName, configJson ], done);
+        },
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE domains', callback);
+};

migrations/20171118000003-apps-add-domain-constraint.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_domain_constraint FOREIGN KEY(domain) REFERENCES domains(domain)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP FOREIGN KEY apps_domain_constraint', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20171118000004-mailboxes-add-domain-constraint.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD CONSTRAINT mailboxes_domain_constraint FOREIGN KEY(domain) REFERENCES domains(domain)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP FOREIGN KEY mailboxes_domain_constraint', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20171118000005-mailboxes-drop-name-constraint.js

@@ -0,0 +1,15 @@
+'use strict';
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP PRIMARY KEY', function (error) {
+      if (error) console.error(error);
+      callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes ADD PRIMARY KEY(name)', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};