Implement tarball based installation

* The base image contains only installer code. Installer code
  can only be changed with a base image change
* The box code is download from s3 instead of git. The s3 tarball
  consists of node dependancies already

Part of #115

installer/scripts/installer.sh

@@ -8,7 +8,7 @@ set -e
 set -x
 
 HOME_DIR="/home/yellowtent"
-SRCDIR="$HOME_DIR/box"
+BOX_SRCDIR="$HOME_DIR/box"
 CONFIG_DIR="$HOME_DIR/config"
 DATA_DIR="$HOME_DIR/data"
 CLOUDRON_SQLITE="$DATA_DIR/cloudron.sqlite"
@@ -51,30 +51,26 @@ if [ -n "$PROVISION_RESTORE_URL" ]; then
 fi
 
 echo "Downloading box versions"
-if [ "$PROVISION_VERSION" = "latest" ]; then
-    REVISION="origin/master"
-else
-    while true; do
-        REVISION=$(curl --retry 5 --retry-delay 5 --max-time 120 -L "$PROVISION_BOX_VERSIONS_URL" | $JSON -D, "$PROVISION_VERSION,revision")
-        [ -n "$REVISION" ] && break
-        echo "Failed to download box versions, trying again"
-    done
-fi
-echo "Updating to revision : $REVISION"
 
-cd "$SRCDIR"
 while true; do
-    timeout 3m git fetch origin && break
-    echo "git fetch timedout, trying again"
-    sleep 2
+    SOURCE_TARBALL_URL=$(curl --retry 5 --retry-delay 5 --max-time 120 -L "$PROVISION_BOX_VERSIONS_URL" | $JSON -D, "$PROVISION_VERSION,sourceTarballUrl")
+    [ -n "$SOURCE_TARBALL_URL" ] && break
+    echo "Failed to download box versions, trying again"
 done
 
-git reset --hard "$REVISION"
+sudo -u yellowtent -H bash <<EOF
+set -e
+set -x
+rm -rf "$BOX_SRCDIR" && mkdir -p "$BOX_SRCDIR"
+echo "Fetching source tarball from $SOURCE_TARBALL_URL"
+curl --retry 5 --retry-delay 5 --max-time 1200 -L "$SOURCE_TARBALL_URL" | tar -zxf - -C "$BOX_SRCDIR"
+cd "$BOX_SRCDIR" && npm rebuild
+EOF
 
 # For the update case, remove any existing config
 rm -rf "$CONFIG_DIR/*"
 
 # https://stackoverflow.com/questions/3348443/a-confusion-about-array-versus-array-in-the-context-of-a-bash-comple
 # Note that this is the latest postinstall.sh
-$SRCDIR/postinstall/postinstall.sh "${SAVED_ARGS[@]}"
+$BOX_SRCDIR/postinstall/postinstall.sh "${SAVED_ARGS[@]}"
 

postinstall/postinstall.sh

@@ -77,15 +77,11 @@ $USER ALL=(root) NOPASSWD: $SRCDIR/installer/scripts/installer.sh
 
 EOF
 
-echo "==== Setup yellowtent ===="
+echo "==== Migrate data ===="
 sudo -u $USER -H bash <<EOF
+set -e
+set -x
 cd $SRCDIR
-while true; do
-    timeout 3m npm install --production && break
-    echo "npm install timedout, trying again"
-    sleep 2
-done
-echo "Migrate data"
 PATH=$PATH:$SRCDIR/node_modules/.bin npm run-script migrate_data
 EOF
 

postinstall/setup_supervisord.sh

@@ -3,7 +3,8 @@
 set -e
 
 USER=yellowtent
-SRCDIR=/home/$USER/box
+BOX_SRCDIR=/home/$USER/box
+INSTALLER_SRCDIR=/home/$USER/installer
 DATA_DIR=/home/$USER/data
 NGINX_CONFIG_DIR=/home/$USER/configs/nginx
 
@@ -28,7 +29,7 @@ EOF
 
 cat > /etc/supervisor/conf.d/box.conf <<EOF
 [program:box]
-command=/usr/bin/node $SRCDIR/app.js
+command=/usr/bin/node $BOX_SRCDIR/app.js
 autostart=true
 autorestart=true
 redirect_stderr=true
@@ -39,9 +40,10 @@ user=yellowtent
 environment=HOME="/home/yellowtent",CLOUDRON="1",USER="yellowtent",DEBUG="box*,connect-lastmile"
 EOF
 
+# FIXME: Run installer in a separate supervisor instead and let it never die
 cat > /etc/supervisor/conf.d/updater.conf <<EOF
 [program:updater]
-command=/usr/bin/node $SRCDIR/installer/server.js update-mode
+command=/usr/bin/node $INSTALLER_SRCDIR/installer/server.js update-mode
 autostart=true
 autorestart=true
 redirect_stderr=true

release/publish.js

@@ -22,7 +22,7 @@ function publish(versionsFileName) {
     sortedVersions.forEach(function (version, index) {
         if (typeof versionsJson[version].imageId !== 'number') die('version ' + version + ' does not have proper imageId');
         if (versionsJson[version].next !== null && typeof versionsJson[version].next !== 'string') die('version ' + version + ' does not have proper next');
-        if (typeof versionsJson[version].revision !== 'string') die('version ' + version + ' does not have proper revision');
+        if (typeof versionsJson[version].sourceTarballUrl !== 'string') die('version ' + version + ' does not have proper sourceTarballUrl');
 
         var nextVersion = versionsJson[version].next;
         if (nextVersion <= version) die('next version cannot be less than current @' + version);

release/versions-dev.json

@@ -1,7 +1,7 @@
 {
     "0.0.1": {
-        "revision": "origin/master",
-        "imageId": 9795675,
+        "sourceTarballUrl": "https://s3.amazonaws.com/cloudron-releases/box-367cb48889b604bdfae33d07693e2195c331c263.tar.gz",
+        "imageId": 9833946,
         "next": null
     }
 }

scripts/createDigitalOceanImage.sh

@@ -3,6 +3,7 @@
 set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+INSTALLER_SOURCE="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
 if [ -z "$DIGITAL_OCEAN_TOKEN" ]; then
     echo "Script requires DIGITAL_OCEAN_TOKEN env to be set"
@@ -13,7 +14,7 @@ JSON="$SCRIPT_DIR/../node_modules/.bin/json"
 CURL="curl -s -u $DIGITAL_OCEAN_TOKEN:"
 UBUNTU_IMAGE_SLUG="ubuntu-14-04-x64" # ID=5141286
 DATE=`date +%Y-%m-%d-%H%M%S`
-BOX_REVISION=origin/master
+INSTALLER_REVISION=$(git rev-parse HEAD)
 IMAGE_REGIONS=(ams3 sfo1 nyc2)
 BOX_SIZE="512mb"
 
@@ -26,7 +27,7 @@ eval set -- "$ARGS"
 
 while true; do
     case "$1" in
-    --revision) BOX_REVISION="$2";;
+    --revision) INSTALLER_REVISION="$2";;
     --regions) IMAGE_REGIONS=($2);; # parse as whitespace separated array
     --size) BOX_SIZE="$2";;
     --) break;;
@@ -54,7 +55,7 @@ function get_pretty_revision() {
     fi
 }
 
-PRETTY_REVISION=$(get_pretty_revision $BOX_REVISION)
+PRETTY_REVISION=$(get_pretty_revision $INSTALLER_REVISION)
 BOX_NAME="box-$PRETTY_REVISION-$DATE" # remove slashes
 SNAPSHOT_NAME="box-$PRETTY_REVISION-$DATE"
 
@@ -188,8 +189,11 @@ while true; do
     sleep 30
 done
 
+echo "Copying installer source"
+(cd "$INSTALLER_SOURCE_DIR" && git archive --format=tar HEAD) | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SCRIPT_DIR/ssh/id_rsa_yellowtent root@$DROPLET_IP "cat - > /root/installer.tar"
+
 echo "Executing init script"
-if ! ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SCRIPT_DIR/ssh/id_rsa_yellowtent root@$DROPLET_IP "/bin/bash /root/initializeBaseUbuntuImage.sh $BOX_REVISION"; then
+if ! ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $SCRIPT_DIR/ssh/id_rsa_yellowtent root@$DROPLET_IP "/bin/bash /root/initializeBaseUbuntuImage.sh $INSTALLER_REVISION"; then
     echo "Init script failed"
     exit 1
 fi

scripts/initializeBaseUbuntuImage.sh

@@ -6,8 +6,8 @@ USER=yellowtent
 USER_HOME="/home/$USER"
 DATA_DIR="$USER_HOME/data"
 APPDATA=$DATA_DIR/appdata
-SRCDIR=$USER_HOME/box
-BOX_REVISION=$1
+INSTALLER_SOURCE_DIR=$USER_HOME/installer
+INSTALLER_REVISION=$1
 
 echo "==== Create User $USER ===="
 id $USER
@@ -19,7 +19,7 @@ fi
 # now exit on failure
 set -e
 
-echo "== Yellowtent base image preparation ($BOX_REVISION) =="
+echo "== Yellowtent base image preparation (installer revision - $INSTALLER_REVISION) =="
 
 export DEBIAN_FRONTEND=noninteractive
 
@@ -40,11 +40,6 @@ echo "==== Install nodejs ===="
 apt-get -y install nodejs npm
 ln -sf /usr/bin/nodejs /usr/bin/node
 
-
-echo "==== Install git ===="
-apt-get -y install git
-
-
 echo "==== Install docker ===="
 # see http://idolstarastronomer.com/painless-docker.html
 echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
@@ -129,26 +124,13 @@ echo "/root/user_home.img $USER_HOME btrfs loop,nosuid 0 0" >> /etc/fstab
 mount -a
 btrfs subvolume create $USER_HOME/data
 
-echo "==== Cloning box repo ===="
-echo "Cloning the box repo"
-mkdir -p $USER_HOME
-cd $USER_HOME
-git clone http://bootstrap:not4long@yellowtent.girish.in/yellowtent/box.git
-cd $SRCDIR
-git reset --hard $BOX_REVISION
-echo "git HEAD is `git rev-parse HEAD`"
-
-NPM_INSTALL="npm install --production"
-rm -rf ./node_modules
-eval $NPM_INSTALL
-RET=$?
-while [[ $RET -ne 0 ]]; do
-    echo "[EE] npm install failed, try again"
-    rm -rf ./node_modules
-    eval $NPM_INSTALL
-    RET=$?
-done
+echo "==== Extracting installer source ===="
+mkdir -p "$INSTALLER_SOURCE_DIR"
+tar xvf /root/installer.tar -C "$INSTALLER_SOURCE_DIR" && rm /root/installer.tar
+echo "$INSTALLER_REVISION" > "$INSTALLER_SOURCE_DIR/REVISION"
 
+echo "=== Rebuilding npm packages ==="
+cd "$INSTALLER_SOURCE_DIR" && npm install
 
 echo "==== Make the user own his home ===="
 chown $USER:$USER -R /home/$USER
@@ -197,26 +179,12 @@ echo "==== Install init script ===="
 cat > /etc/init.d/cloudron-bootstrap <<EOF
 #!/bin/bash
 
-checkout_installer() {
-    cd "$SRCDIR"
-    while true; do
-        timeout 3m git fetch origin && break
-        echo "git fetch timedout, trying again"
-        sleep 2
-    done
-
-    git reset --hard "$1"
-}
-
 do_start() {
-    # this hack lets us work with refs instead of revisions
-    checkout_installer "$BOX_REVISION"
-
     mkdir -p /var/log/cloudron
 
     exec 2>&1 1> "/var/log/cloudron/bootstrap.log"
 
-    DEBUG="box*,connect-lastmile" $SRCDIR/installer/server.js provision-mode 2>&1 1> /var/log/cloudron/installserver.log &
+    DEBUG="box*,connect-lastmile" $INSTALLER_SOURCE_DIR/installer/server.js provision-mode 2>&1 1> /var/log/cloudron/installserver.log &
 
     echo "Disabling cloudron-bootstrap init script"
     update-rc.d cloudron-bootstrap remove