diff --git a/.gitattributes b/.gitattributes index d79789d5e..09fd19fa8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,8 @@ -# Skip files when using git archive +# following files are skipped when exporting using git archive +/release export-ignore +/admin export-ignore +test export-ignore .gitattributes export-ignore .gitignore export-ignore /scripts export-ignore -test export-ignore diff --git a/.gitignore b/.gitignore index a262f1358..3556d45db 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,9 @@ coverage/ docs/ webadmin/dist/ setup/splash/website/ +installer/src/certs/server.key # vim swap files *.swp + diff --git a/installer/images/createBoxTarball b/installer/images/createBoxTarball new file mode 100755 index 000000000..1303475ea --- /dev/null +++ b/installer/images/createBoxTarball @@ -0,0 +1,118 @@ +#!/bin/bash + +set -eu + +assertNotEmpty() { + : "${!1:? "$1 is not set."}" +} + +# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt +# brew install gnu-getopt to get the GNU getopt on OS X +[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt" +readonly GNU_GETOPT + +args=$(${GNU_GETOPT} -o "" -l "revision:,output:,publish,no-upload" -n "$0" -- "$@") +eval set -- "${args}" + +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly box_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../box" && pwd)" + +delete_bundle="yes" +commitish="HEAD" +publish="no" +upload="yes" +bundle_file="" + +while true; do + case "$1" in + --revision) commitish="$2"; shift 2;; + --output) bundle_file="$2"; delete_bundle="no"; shift 2;; + --no-upload) upload="no"; shift;; + --publish) publish="yes"; shift;; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac +done + +if [[ "${upload}" == "no" && "${publish}" == "yes" ]]; then + echo "Cannot publish without uploading" + exit 1 +fi + +readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint? + +assertNotEmpty AWS_DEV_ACCESS_KEY +assertNotEmpty AWS_DEV_SECRET_KEY + +if ! $(cd "${box_dir}" && git diff --exit-code >/dev/null); then + echo "You have local changes, stash or commit them to proceed" + exit 1 +fi + +version=$(cd "${box_dir}" && git rev-parse "${commitish}") +bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR) +[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${version}.tar.gz" + +chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group +echo "Checking out code [${version}] into ${bundle_dir}" +(cd "${box_dir}" && git archive --format=tar ${version} | (cd "${bundle_dir}" && tar xf -)) + +if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then + echo "Reusing dev modules from cache" + cp -r "${TMPDIR}/boxtarball.cache/node_modules-all/." "${bundle_dir}/node_modules" +else + echo "Installing modules with dev dependencies" + (cd "${bundle_dir}" && npm install) + + echo "Caching dev dependencies" + mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-all" + rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-all/" + cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.all" +fi + +echo "Building webadmin assets" +(cd "${bundle_dir}" && gulp) + +echo "Remove intermediate files required at build-time only" +rm -rf "${bundle_dir}/node_modules/" +rm -rf "${bundle_dir}/webadmin/src" +rm -rf "${bundle_dir}/gulpfile.js" + +if diff "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod" "${bundle_dir}/npm-shrinkwrap.json" >/dev/null 2>&1; then + echo "Reusing prod modules from cache" + cp -r "${TMPDIR}/boxtarball.cache/node_modules-prod/." "${bundle_dir}/node_modules" +else + echo "Installing modules for production" + (cd "${bundle_dir}" && npm install --production --no-optional) + + echo "Caching prod dependencies" + mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-prod" + rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-prod/" + cp "${bundle_dir}/npm-shrinkwrap.json" "${TMPDIR}/boxtarball.cache/npm-shrinkwrap.json.prod" +fi + +echo "Create final tarball" +(cd "${bundle_dir}" && tar czf "${bundle_file}" .) +echo "Cleaning up ${bundle_dir}" +rm -rf "${bundle_dir}" + +if [[ "${upload}" == "yes" ]]; then + echo "Uploading bundle to S3" + # That special header is needed to allow access with singed urls created with different aws credentials than the ones the file got uploaded + s3cmd --multipart-chunk-size-mb=5 --ssl --acl-public --access_key="${AWS_DEV_ACCESS_KEY}" --secret_key="${AWS_DEV_SECRET_KEY}" --no-mime-magic put "${bundle_file}" "s3://dev-cloudron-releases/box-${version}.tar.gz" + + versions_file_url="https://dev-cloudron-releases.s3.amazonaws.com/box-${version}.tar.gz" + echo "The URL for the versions file is: ${versions_file_url}" + + if [[ "${publish}" == "yes" ]]; then + echo "Publishing to dev" + ${script_dir}/release/release create --env dev --code "${versions_file_url}" + fi +fi + +if [[ "${delete_bundle}" == "no" ]]; then + echo "Tarball preserved at ${bundle_file}" +else + rm "${bundle_file}" +fi + diff --git a/installer/images/createImage b/installer/images/createImage new file mode 100755 index 000000000..398ded5b3 --- /dev/null +++ b/installer/images/createImage @@ -0,0 +1,195 @@ +#!/bin/bash + +set -eu -o pipefail + +assertNotEmpty() { + : "${!1:? "$1 is not set."}" +} + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly INSTALLER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" +export JSON="${INSTALLER_DIR}/node_modules/.bin/json" + +provider="digitalocean" +installer_revision=$(git rev-parse HEAD) +box_name="" +server_id="" +server_ip="" +destroy_server="yes" +deploy_env="dev" + +# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt +# brew install gnu-getopt to get the GNU getopt on OS X +[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt" +readonly GNU_GETOPT + +args=$(${GNU_GETOPT} -o "" -l "provider:,revision:,regions:,size:,name:,no-destroy,env:" -n "$0" -- "$@") +eval set -- "${args}" + +while true; do + case "$1" in + --env) deploy_env="$2"; shift 2;; + --revision) installer_revision="$2"; shift 2;; + --provider) provider="$2"; shift 2;; + --name) box_name="$2"; destroy_server="no"; shift 2;; + --no-destroy) destroy_server="no"; shift 2;; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac +done + +echo "Creating image using ${provider}" +if [[ "${provider}" == "digitalocean" ]]; then + if [[ "${deploy_env}" == "staging" ]]; then + assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING + export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}" + elif [[ "${deploy_env}" == "dev" ]]; then + assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV + export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}" + elif [[ "${deploy_env}" == "prod" ]]; then + assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD + export DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}" + else + echo "No such env ${deploy_env}." + exit 1 + fi + + vps="${SCRIPT_DIR}/digitalocean.sh" +elif [[ "${provider}" == "vultr" ]]; then + export VULTR_TOKEN="${VULTR_TOKEN}" + vps="${SCRIPT_DIR}/vultr.js" +else + echo "Unknown provider : ${provider}" + exit 1 +fi + +readonly ssh_keys="${HOME}/.ssh/id_rsa_caas_${deploy_env}" +readonly scp202="scp -P 202 -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}" +readonly scp22="scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}" + +readonly ssh202="ssh -p 202 -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}" +readonly ssh22="ssh -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i ${ssh_keys}" + +if [[ ! -f "${ssh_keys}" ]]; then + echo "caas ssh key is missing at ${ssh_keys} (pick it up from secrets repo)" + exit 1 +fi + +function get_pretty_revision() { + local git_rev="$1" + local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null) + + echo "${sha1}" +} + +now=$(date "+%Y-%m-%d-%H%M%S") +pretty_revision=$(get_pretty_revision "${installer_revision}") + +if [[ -z "${box_name}" ]]; then + # if you change this, change the regexp is appstore/janitor.js + box_name="box-${deploy_env}-${pretty_revision}-${now}" # remove slashes + + # create a new server if no name given + if ! caas_ssh_key_id=$($vps get_ssh_key_id "caas"); then + echo "Could not query caas ssh key" + exit 1 + fi + echo "Detected caas ssh key id: ${caas_ssh_key_id}" + + echo "Creating Server with name [${box_name}]" + if ! server_id=$($vps create ${caas_ssh_key_id} ${box_name}); then + echo "Failed to create server" + exit 1 + fi + echo "Created server with id: ${server_id}" + + # If we run scripts overenthusiastically without the wait, setup script randomly fails + echo -n "Waiting 120 seconds for server creation" + for i in $(seq 1 24); do + echo -n "." + sleep 5 + done + echo "" +else + if ! server_id=$($vps get_id "${box_name}"); then + echo "Could not determine id from name" + exit 1 + fi + echo "Reusing server with id: ${server_id}" + + $vps power_on "${server_id}" +fi + +# Query until we get an IP +while true; do + echo "Trying to get the server IP" + if server_ip=$($vps get_ip "${server_id}"); then + echo "Server IP : [${server_ip}]" + break + fi + echo "Timedout, trying again in 10 seconds" + sleep 10 +done + +while true; do + echo "Trying to copy init script to server" + if $scp22 "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" root@${server_ip}:.; then + break + fi + echo "Timedout, trying again in 30 seconds" + sleep 30 +done + +echo "Copying INFRA_VERSION" +$scp22 "${SCRIPT_DIR}/../../box/setup/INFRA_VERSION" root@${server_ip}:. + +echo "Copying installer source" +cd "${INSTALLER_DIR}" +git archive --format=tar HEAD | $ssh22 "root@${server_ip}" "cat - > /root/installer.tar" + +echo "Executing init script" +if ! $ssh22 "root@${server_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then + echo "Init script failed" + exit 1 +fi + +echo "Copy over certs" +cd "${SCRIPT_DIR}/../../secrets" +blackbox_cat installer/server.crt.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/server.crt" +blackbox_cat installer/server.key.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/server.key" +blackbox_cat installer_ca/ca.crt.gpg | $ssh202 "root@${server_ip}" "cat - > /home/yellowtent/installer/src/certs/ca.crt" + +echo "Shutting down server with id : ${server_id}" +$ssh202 "root@${server_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail + +# wait 10 secs for actual shutdown +echo "Waiting for 10 seconds for server to shutdown" +sleep 30 + +echo "Powering off server" +if ! $vps power_off "${server_id}"; then + echo "Could not power off server" + exit 1 +fi + +snapshot_name="box-${deploy_env}-${pretty_revision}-${now}" +echo "Snapshotting as ${snapshot_name}" +if ! image_id=$($vps snapshot "${server_id}" "${snapshot_name}"); then + echo "Could not snapshot and get image id" + exit 1 +fi + +if [[ "${destroy_server}" == "yes" ]]; then + echo "Destroying server" + if ! $vps destroy "${server_id}"; then + echo "Could not destroy server" + exit 1 + fi +else + echo "Skipping server destroy" +fi + +echo "Transferring image ${image_id} to other regions" +$vps transfer_image_to_all_regions "${image_id}" + +echo "Done." diff --git a/installer/images/digitalocean.sh b/installer/images/digitalocean.sh new file mode 100755 index 000000000..f770d7466 --- /dev/null +++ b/installer/images/digitalocean.sh @@ -0,0 +1,240 @@ +#!/bin/bash + +if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then + echo "Script requires DIGITAL_OCEAN_TOKEN env to be set" + exit 1 +fi + +if [[ -z "${JSON}" ]]; then + echo "Script requires JSON env to be set to path of JSON binary" + exit 1 +fi + +readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:" + +function debug() { + echo "$@" >&2 +} + +function get_ssh_key_id() { + id=$($CURL "https://api.digitalocean.com/v2/account/keys" \ + | $JSON ssh_keys \ + | $JSON -c "this.name === \"$1\"" \ + | $JSON 0.id) + [[ -z "$id" ]] && exit 1 + echo "$id" +} + +function create_droplet() { + local ssh_key_id="$1" + local box_name="$2" + + local image_region="sfo1" + local ubuntu_image_slug="ubuntu-15-04-x64" # id=12658446 + local box_size="512mb" + + local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}" + + id=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id) + [[ -z "$id" ]] && exit 1 + echo "$id" +} + +function get_droplet_ip() { + local droplet_id="$1" + ip=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address") + [[ -z "$ip" ]] && exit 1 + echo "$ip" +} + +function get_droplet_id() { + local droplet_name="$1" + id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id") + [[ -z "$id" ]] && exit 1 + echo "$id" +} + +function power_off_droplet() { + local droplet_id="$1" + local data='{"type":"power_off"}' + local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions") + local event_id=`echo "${response}" | $JSON action.id` + + if [[ -z "${event_id}" ]]; then + debug "Got no event id, assuming already powered off." + debug "Response: ${response}" + return + fi + + debug "Powered off droplet. Event id: ${event_id}" + debug -n "Waiting for droplet to power off" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + debug -n "." + sleep 10 + done + debug "" +} + +function power_on_droplet() { + local droplet_id="$1" + local data='{"type":"power_on"}' + local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id` + + debug "Powered on droplet. Event id: ${event_id}" + + if [[ -z "${event_id}" ]]; then + debug "Got no event id, assuming already powered on" + return + fi + + debug -n "Waiting for droplet to power on" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + debug -n "." + sleep 10 + done + debug "" +} + +function get_image_id() { + local snapshot_name="$1" + local image_id="" + + image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \ + | $JSON images \ + | $JSON -c "this.name === \"${snapshot_name}\"" 0.id) + + if [[ -n "${image_id}" ]]; then + echo "${image_id}" + fi +} + +function snapshot_droplet() { + local droplet_id="$1" + local snapshot_name="$2" + local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}" + local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id` + + debug "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}" + debug -n "Waiting for snapshot to complete" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + debug -n "." + sleep 10 + done + debug "" + + get_image_id "${snapshot_name}" +} + +function destroy_droplet() { + local droplet_id="$1" + # TODO: check for 204 status + $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}" + debug "Droplet destroyed" + debug "" +} + +function transfer_image() { + local image_id="$1" + local region_slug="$2" + local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}" + local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id` + echo "${event_id}" +} + +function wait_for_image_event() { + local image_id="$1" + local event_id="$2" + + debug -n "Waiting for ${event_id}" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + debug -n "." + sleep 10 + done + debug "" +} + +function transfer_image_to_all_regions() { + local image_id="$1" + + xfer_events=() + image_regions=(ams3) ## sfo1 is where the image is created + for image_region in ${image_regions[@]}; do + xfer_event=$(transfer_image ${image_id} ${image_region}) + echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}" + xfer_events+=("${xfer_event}") + sleep 1 + done + + echo "Image transfer initiated, but they will take some time to get transferred." + + for xfer_event in ${xfer_events[@]}; do + $vps wait_for_image_event "${image_id}" "${xfer_event}" + done +} + +if [[ $# -lt 1 ]]; then + debug " " + exit 1 +fi + +case $1 in +get_ssh_key_id) + get_ssh_key_id "${@:2}" + ;; + +create) + create_droplet "${@:2}" + ;; + +get_id) + get_droplet_id "${@:2}" + ;; + +get_ip) + get_droplet_ip "${@:2}" + ;; + +power_on) + power_on_droplet "${@:2}" + ;; + +power_off) + power_off_droplet "${@:2}" + ;; + +snapshot) + snapshot_droplet "${@:2}" + ;; + +destroy) + destroy_droplet "${@:2}" + ;; + +transfer_image_to_all_regions) + transfer_image_to_all_regions "${@:2}" + ;; + +*) + echo "Unknown command $1" + exit 1 +esac diff --git a/installer/images/images b/installer/images/images new file mode 100755 index 000000000..dcecad1c9 --- /dev/null +++ b/installer/images/images @@ -0,0 +1,200 @@ +#!/usr/bin/env node + +'use strict'; + +require('supererror')({ splatchError: true }); + +var superagent = require('superagent'), + async = require('async'), + yesno = require('yesno'), + program = require('commander'); + +var DIGITALOCEAN = 'https://api.digitalocean.com/v2'; + +var ENVIRONMENTS = { + 'dev': { + tag: 'dev', + releaseUrl: 'https://s3.amazonaws.com/dev-cloudron-releases/versions.json', + digitalOceanToken: process.env.DIGITAL_OCEAN_TOKEN_DEV + }, + 'staging': { + tag: 'staging', + releaseUrl: 'https://s3.amazonaws.com/staging-cloudron-releases/versions.json', + digitalOceanToken: process.env.DIGITAL_OCEAN_TOKEN_STAGING + }, + 'prod': { + tag: 'prod', + releaseUrl: 'https://s3.amazonaws.com/prod-cloudron-releases/versions.json', + digitalOceanToken: process.env.DIGITAL_OCEAN_TOKEN_PROD + } +}; + +function deleteImage(image, token, callback) { + var url = DIGITALOCEAN + '/images/' + image.id; + + console.log('Deleting image %s ...', image.name); + + superagent.del(url).set('Authorization', 'Bearer ' + token).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + callback(null); + }); +} + +function getImages(token, callback) { + var images = []; + var nextPage = DIGITALOCEAN + '/images?private=true'; + + async.doWhilst(function (callback) { + superagent.get(nextPage).set('Authorization', 'Bearer ' + token).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + nextPage = (result.body.links && result.body.links.pages && nextPage !== result.body.links.pages.next) ? result.body.links.pages.next : null; + images = images.concat(result.body.images); + + callback(null); + }); + }, function () { return !!nextPage; }, function (error) { + if (error) return callback(error); + + callback(null, images); + }); +} + +function printImages(env, images, releases, callback) { + console.log(''); + console.log('%s:', env.tag); + console.log(''); + + var imageRegExp = new RegExp('box-(?:dev|staging|prod)-[0-9,a-f]{7}-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}'); + + images.forEach(function (image) { + if (!imageRegExp.test(image.name)) return; + + var releaseNumber = []; + for (var release in releases) { + if (releases.hasOwnProperty(release)) { + if (releases[release].imageId === image.id) { + releaseNumber.push(release); + } + } + } + + console.log(' %s : %s %s\t[%s]', image.id, image.name, releaseNumber.length ? releaseNumber.join(', ') : ' ', image.regions); + }); + + console.log(''); + + callback(null); +} + +function deleteUnusedImages(env, images, releases, callback) { + console.log(''); + console.log('Cleanup images on %s:', env.tag); + + var imagesToCleanup = []; + + var imageRegExp = new RegExp('box-(?:dev|staging|prod)-[0-9,a-f]{7}-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}'); + + images.forEach(function (image) { + if (!imageRegExp.test(image.name)) return; + + for (var release in releases) { + if (releases.hasOwnProperty(release)) { + if (releases[release].imageId === image.id) { + return; + } + } + } + + // we reached here so no release found + imagesToCleanup.push(image); + }); + + if (imagesToCleanup.length === 0) { + console.log('All images belong to a release.'); + return callback(null); + } + + imagesToCleanup.forEach(function (image) { + console.log(' %s : %s [%s]', image.id, image.name, image.regions); + }); + + console.log(''); + + yesno.ask('Do you want to delete those images? [y/N]', false, function (ok) { + if (ok) { + async.each(imagesToCleanup, function (image, callback) { + deleteImage(image, env.digitalOceanToken, callback); + }, callback); + return; + } + + callback(null); + }); +} + +function exit(error) { + if (error) console.error(error); + process.exit(error ? 1 : 0); +} + +function listImages(options) { + var env = ENVIRONMENTS[options.env]; + + getImages(env.digitalOceanToken, function (error, result) { + if (error) return exit(error); + + var images = result; + superagent.get(env.releaseUrl).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + // we get it as text + var releases = JSON.parse(result.text); + + printImages(env, images, releases, exit); + }); + }); +} + +function cleanupImages(options) { + var env = ENVIRONMENTS[options.env]; + + getImages(env.digitalOceanToken, function (error, result) { + if (error) return exit(error); + + var images = result; + superagent.get(env.releaseUrl).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + // we get it as text + var releases = JSON.parse(result.text); + + deleteUnusedImages(env, images, releases, exit); + }); + }); +} + +program.version('0.0.1'); + +program.command('list') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .description('List images of environment') + .action(listImages); + +program.command('cleanup') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .description('Cleanup images of environment') + .action(cleanupImages); + +program.parse(process.argv); + +if (!process.argv.slice(2).length) { + program.outputHelp(); +} else { // https://github.com/tj/commander.js/issues/338 + var knownCommand = program.commands.some(function (command) { return command._name === process.argv[2]; }); + if (!knownCommand) { + console.error('Unknown command: ' + process.argv[2]); + process.exit(1); + } +} diff --git a/installer/images/initializeBaseUbuntuImage.sh b/installer/images/initializeBaseUbuntuImage.sh new file mode 100755 index 000000000..26b2a0026 --- /dev/null +++ b/installer/images/initializeBaseUbuntuImage.sh @@ -0,0 +1,306 @@ +#!/bin/bash + +set -euv -o pipefail + +readonly USER=yellowtent +readonly USER_HOME="/home/${USER}" +readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer" +readonly INSTALLER_REVISION="$1" +readonly SELFHOSTED=$(( $# > 1 ? 1 : 0 )) +readonly USER_DATA_FILE="/root/user_data.img" +readonly USER_DATA_DIR="/home/yellowtent/data" + +readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then + source "${SOURCE_DIR}/INFRA_VERSION" +else + echo "No INFRA_VERSION found, skip pulling docker images" +fi + +if [ ${SELFHOSTED} == 0 ]; then + echo "!! Initializing Ubuntu image for CaaS" +else + echo "!! Initializing Ubuntu image for Selfhosting" +fi + +echo "==== Create User ${USER} ====" +if ! id "${USER}"; then + useradd "${USER}" -m +fi + +echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ===" + +export DEBIAN_FRONTEND=noninteractive + +echo "=== Upgrade ===" +apt-get update +apt-get upgrade -y +apt-get install -y curl + +# Setup firewall before everything. docker creates it's own chain and the -X below will remove it +# Do NOT use iptables-persistent because it's startup ordering conflicts with docker +echo "=== Setting up firewall ===" +# clear tables and set default policy +iptables -F # flush all chains +iptables -X # delete all chains +# default policy for filter table +iptables -P INPUT DROP +iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject +iptables -P OUTPUT ACCEPT + +# NOTE: keep these in sync with src/apps.js validatePortBindings +# allow ssh, http, https, ping, dns +iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT +if [ ${SELFHOSTED} == 0 ]; then + iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT +else + iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT +fi +iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT +iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT +iptables -A INPUT -p udp --sport 53 -j ACCEPT +iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP: + +# loopback +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT + +# prevent DoS +# iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT + +# log dropped incoming. keep this at the end of all the rules +iptables -N LOGGING # new chain +iptables -A INPUT -j LOGGING # last rule in INPUT chain +iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7 +iptables -A LOGGING -j DROP + +echo "==== Install btrfs tools ===" +apt-get -y install btrfs-tools + +echo "==== Install docker ====" +# install docker from binary to pin it to a specific version. the current debian repo does not allow pinning +curl https://get.docker.com/builds/Linux/x86_64/docker-1.9.1 > /usr/bin/docker +chmod +x /usr/bin/docker +groupadd docker +cat > /etc/systemd/system/docker.socket < /etc/systemd/system/docker.service <> /etc/fstab +mkdir -p "${USER_DATA_DIR}" && mount "${USER_DATA_FILE}" + +systemctl daemon-reload +systemctl enable docker +systemctl start docker + +# give docker sometime to start up and create iptables rules +# those rules come in after docker has started, and we want to wait for them to be sure iptables-save has all of them +sleep 10 + +# Disable forwarding to metadata route from containers +iptables -I FORWARD -d 169.254.169.254 -j DROP + +# ubuntu will restore iptables from this file automatically. this is here so that docker's chain is saved to this file +mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4 + +echo "=== Enable memory accounting ==" +sed -e 's/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1 panic_on_oops=1 panic=5"/' -i /etc/default/grub +update-grub + +# now add the user to the docker group +usermod "${USER}" -a -G docker + +if [ -z $(echo "${INFRA_VERSION}") ]; then + echo "Skip pulling base docker images" +else + echo "=== Pulling base docker images ===" + docker pull "${BASE_IMAGE}" + + echo "=== Pulling mysql addon image ===" + docker pull "${MYSQL_IMAGE}" + + echo "=== Pulling postgresql addon image ===" + docker pull "${POSTGRESQL_IMAGE}" + + echo "=== Pulling redis addon image ===" + docker pull "${REDIS_IMAGE}" + + echo "=== Pulling mongodb addon image ===" + docker pull "${MONGODB_IMAGE}" + + echo "=== Pulling graphite docker images ===" + docker pull "${GRAPHITE_IMAGE}" + + echo "=== Pulling mail relay ===" + docker pull "${MAIL_IMAGE}" +fi + +echo "==== Install nginx ====" +apt-get -y install nginx-full + +echo "==== Install build-essential ====" +apt-get -y install build-essential rcconf + + +echo "==== Install mysql ====" +debconf-set-selections <<< 'mysql-server mysql-server/root_password password password' +debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password' +apt-get -y install mysql-server + +echo "==== Install pwgen ====" +apt-get -y install pwgen + +echo "==== Install collectd ===" +if ! apt-get install -y collectd collectd-utils; then + # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this + echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html" + sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf +fi +update-rc.d -f collectd remove + +# this simply makes it explicit that we run logrotate via cron. it's already part of base ubuntu +echo "==== Install logrotate ===" +apt-get install -y cron logrotate +systemctl enable cron + +echo "==== Extracting installer source ====" +rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}" +tar xvf /root/installer.tar -C "${INSTALLER_SOURCE_DIR}" && rm /root/installer.tar +echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION" + +echo "==== Install nodejs ====" +# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803 +mkdir -p /usr/local/node-4.1.1 +curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1 +ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node +ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm +apt-get install -y python # Install python which is required for npm rebuild + +echo "=== Rebuilding npm packages ===" +cd "${INSTALLER_SOURCE_DIR}" && npm install --production +chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}" + +echo "==== Install installer systemd script ====" +provisionEnv="PROVISION=digitalocean" +if [ ${SELFHOSTED} == 1 ]; then + provisionEnv="PROVISION=local" +fi + +cat > /etc/systemd/system/cloudron-installer.service < /etc/systemd/system/iptables-restore.service < /etc/systemd/system/box-setup.service < ')); +} + +function getSshKeyId(keyName, callback) { + var res = request.get('https://api.vultr.com/v1/sshkey/list') + .query({ api_key : gApiToken }) + .end(); + + if (res.statusCode !== 200) exit(new Error('Invalid response')); + + var allKeyIds = Object.keys(res.body); + for (var i = 0; i < allKeyIds.length; i++) { + if (keyName === res.body[allKeyIds[i]].name) return callback(null, allKeyIds[i]); // also SSHKEYID + } + + callback(new Error('key not found')); +} + +function create(keyId, name, callback) { + var regionId = 5; // LA (https://api.vultr.com/v1/regions/list) + var planId = 29; // 768MB RAM (https://api.vultr.com/v1/regions/list) + var osid = 191; // Ubuntu 15.04 x64 (see https://api.vultr.com/v1/os/list). 15.04 has some systemd issue + + var res = request.post('https://api.vultr.com/v1/server/create') + .query({ api_key : gApiToken }) + .type('form') + .send({ DCID: regionId, VPSPLANID: planId, OSID : osid, label: name, SSHKEYID: keyId }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid response creating server')); + + return callback(null, res.body.SUBID); +} + +function getIp(id, callback) { + var res = request.get('https://api.vultr.com/v1/server/list') + .query({ api_key : gApiToken, SUBID: id }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid statusCode querying IP')); + + var info = res.body; + if (!info) return callback(new Error('Invalid response querying IP')); + + if (info.power_status !== 'running' || info.server_state !== 'ok' || info.status !== 'active') return callback(new Error('Server is not up yet')); + + return callback(null, info.main_ip); +} + +function getId(name, callback) { + var res = request.get('https://api.vultr.com/v1/server/list') + .query({ api_key : gApiToken }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid statusCode querying id')); + + var serverIds = Object.keys(res.body); + for (var i = 0; i < serverIds.length; i++) { + if (res.body[serverIds[i]].label === name) return callback(null, serverIds[0]); + } + + callback(new Error('no server with id found')); +} + +function powerOn(id, callback) { + var res = request.post('https://api.vultr.com/v1/server/start') + .query({ api_key : gApiToken }) + .type('form') + .send({ SUBID: id }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid statusCode powering on')); + + callback(null); +} + +function powerOff(id, callback) { + var res = request.post('https://api.vultr.com/v1/server/halt') + .query({ api_key : gApiToken }) + .type('form') + .send({ SUBID: id }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid statusCode powering off')); + + callback(null); +} + +function waitForSnapshot(id) { + var res = request.get('https://api.vultr.com/v1/snapshot/list') + .query({ api_key : gApiToken }) + .end(); + + if (res.statusCode !== 200) { + console.error('Invalid statusCode waiting for snapshot'); + return false; + } + + if (res.body[id].status === 'complete') return true; + + console.error('snapshot not complete : ' + res.body[id].status); + + return false; +} + +function snapshot(id, name, callback) { + var res = request.post('https://api.vultr.com/v1/snapshot/create') + .query({ api_key : gApiToken }) + .type('form') + .send({ SUBID: id, description: name }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid statusCode powering off')); + + for (var i = 0; i < 200; i++) { + if (waitForSnapshot(res.body.SNAPSHOTID)) break; + sleep(10); + } + + callback(null, res.body.SNAPSHOTID); +} + +function destroy(id, callback) { + var res = request.post('https://api.vultr.com/v1/server/destroy') + .query({ api_key : gApiToken }) + .type('form') + .send({ SUBID: id }) + .end(); + + if (res.statusCode !== 200) return callback(new Error('Invalid statusCode powering off')); + + callback(); +} + +switch (process.argv[2]) { +case 'get_ssh_key_id': + getSshKeyId(process.argv[3], exit); + break; + +case 'create': + create(process.argv[3], process.argv[4], exit); + break; + +case 'get_ip': + getIp(process.argv[3], exit); + break; + +case 'get_id': + getId(process.argv[3], exit); + break; + +case 'power_on': + powerOn(process.argv[3], exit); + break; + +case 'power_off': + powerOff(process.argv[3], exit); + break; + +case 'snapshot': + snapshot(process.argv[3], process.argv[4], exit); + break; + +case 'destroy': + destroy(process.argv[3], exit); + break; + +case 'transfer_image_to_all_regions': + // nothing to do? + break; + +default: + exit(new Error('Unknown command:' + process.argv[2])); +} diff --git a/installer/npm-shrinkwrap.json b/installer/npm-shrinkwrap.json new file mode 100644 index 000000000..c50935343 --- /dev/null +++ b/installer/npm-shrinkwrap.json @@ -0,0 +1,516 @@ +{ + "name": "installer", + "version": "0.0.1", + "dependencies": { + "async": { + "version": "1.5.0", + "from": "async@>=1.5.0 <2.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.0.tgz" + }, + "body-parser": { + "version": "1.14.1", + "from": "body-parser@>=1.12.0 <2.0.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.14.1.tgz", + "dependencies": { + "bytes": { + "version": "2.1.0", + "from": "bytes@2.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-2.1.0.tgz" + }, + "content-type": { + "version": "1.0.1", + "from": "content-type@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz" + }, + "depd": { + "version": "1.1.0", + "from": "depd@>=1.1.0 <1.2.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz" + }, + "http-errors": { + "version": "1.3.1", + "from": "http-errors@>=1.3.1 <1.4.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz", + "dependencies": { + "inherits": { + "version": "2.0.1", + "from": "inherits@>=2.0.1 <2.1.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, + "statuses": { + "version": "1.2.1", + "from": "statuses@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz" + } + } + }, + "iconv-lite": { + "version": "0.4.12", + "from": "iconv-lite@0.4.12", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.12.tgz" + }, + "on-finished": { + "version": "2.3.0", + "from": "on-finished@>=2.3.0 <2.4.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.1", + "from": "ee-first@1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + } + } + }, + "qs": { + "version": "5.1.0", + "from": "qs@5.1.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-5.1.0.tgz" + }, + "raw-body": { + "version": "2.1.4", + "from": "raw-body@>=2.1.4 <2.2.0", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.1.4.tgz", + "dependencies": { + "unpipe": { + "version": "1.0.0", + "from": "unpipe@1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + } + } + }, + "type-is": { + "version": "1.6.9", + "from": "type-is@>=1.6.9 <1.7.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz", + "dependencies": { + "media-typer": { + "version": "0.3.0", + "from": "media-typer@0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "mime-types": { + "version": "2.1.7", + "from": "mime-types@>=2.1.7 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz", + "dependencies": { + "mime-db": { + "version": "1.19.0", + "from": "mime-db@>=1.19.0 <1.20.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz" + } + } + } + } + } + } + }, + "connect-lastmile": { + "version": "0.0.13", + "from": "connect-lastmile@0.0.13", + "resolved": "https://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.13.tgz", + "dependencies": { + "debug": { + "version": "2.1.3", + "from": "debug@>=2.1.0 <2.2.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz", + "dependencies": { + "ms": { + "version": "0.7.0", + "from": "ms@0.7.0", + "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz" + } + } + } + } + }, + "debug": { + "version": "2.2.0", + "from": "debug@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", + "dependencies": { + "ms": { + "version": "0.7.1", + "from": "ms@0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" + } + } + }, + "express": { + "version": "4.13.3", + "from": "express@>=4.11.2 <5.0.0", + "resolved": "https://registry.npmjs.org/express/-/express-4.13.3.tgz", + "dependencies": { + "accepts": { + "version": "1.2.13", + "from": "accepts@>=1.2.12 <1.3.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz", + "dependencies": { + "mime-types": { + "version": "2.1.7", + "from": "mime-types@>=2.1.6 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz", + "dependencies": { + "mime-db": { + "version": "1.19.0", + "from": "mime-db@>=1.19.0 <1.20.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz" + } + } + }, + "negotiator": { + "version": "0.5.3", + "from": "negotiator@0.5.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz" + } + } + }, + "array-flatten": { + "version": "1.1.1", + "from": "array-flatten@1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + }, + "content-disposition": { + "version": "0.5.0", + "from": "content-disposition@0.5.0", + "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz" + }, + "content-type": { + "version": "1.0.1", + "from": "content-type@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz" + }, + "cookie": { + "version": "0.1.3", + "from": "cookie@0.1.3", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.3.tgz" + }, + "cookie-signature": { + "version": "1.0.6", + "from": "cookie-signature@1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + }, + "depd": { + "version": "1.0.1", + "from": "depd@>=1.0.1 <1.1.0", + "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz" + }, + "escape-html": { + "version": "1.0.2", + "from": "escape-html@1.0.2", + "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.2.tgz" + }, + "etag": { + "version": "1.7.0", + "from": "etag@>=1.7.0 <1.8.0", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz" + }, + "finalhandler": { + "version": "0.4.0", + "from": "finalhandler@0.4.0", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.4.0.tgz", + "dependencies": { + "unpipe": { + "version": "1.0.0", + "from": "unpipe@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + } + } + }, + "fresh": { + "version": "0.3.0", + "from": "fresh@0.3.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz" + }, + "merge-descriptors": { + "version": "1.0.0", + "from": "merge-descriptors@1.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz" + }, + "methods": { + "version": "1.1.1", + "from": "methods@>=1.1.1 <1.2.0", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz" + }, + "on-finished": { + "version": "2.3.0", + "from": "on-finished@>=2.3.0 <2.4.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.1", + "from": "ee-first@1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + } + } + }, + "parseurl": { + "version": "1.3.0", + "from": "parseurl@>=1.3.0 <1.4.0", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz" + }, + "path-to-regexp": { + "version": "0.1.7", + "from": "path-to-regexp@0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + }, + "proxy-addr": { + "version": "1.0.8", + "from": "proxy-addr@>=1.0.8 <1.1.0", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.8.tgz", + "dependencies": { + "forwarded": { + "version": "0.1.0", + "from": "forwarded@>=0.1.0 <0.2.0", + "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz" + }, + "ipaddr.js": { + "version": "1.0.1", + "from": "ipaddr.js@1.0.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.1.tgz" + } + } + }, + "qs": { + "version": "4.0.0", + "from": "qs@4.0.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz" + }, + "range-parser": { + "version": "1.0.3", + "from": "range-parser@>=1.0.2 <1.1.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz" + }, + "send": { + "version": "0.13.0", + "from": "send@0.13.0", + "resolved": "http://registry.npmjs.org/send/-/send-0.13.0.tgz", + "dependencies": { + "destroy": { + "version": "1.0.3", + "from": "destroy@1.0.3", + "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz" + }, + "http-errors": { + "version": "1.3.1", + "from": "http-errors@>=1.3.1 <1.4.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz", + "dependencies": { + "inherits": { + "version": "2.0.1", + "from": "inherits@>=2.0.1 <2.1.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + } + } + }, + "mime": { + "version": "1.3.4", + "from": "mime@1.3.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz" + }, + "ms": { + "version": "0.7.1", + "from": "ms@0.7.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" + }, + "statuses": { + "version": "1.2.1", + "from": "statuses@>=1.2.1 <1.3.0", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz" + } + } + }, + "serve-static": { + "version": "1.10.0", + "from": "serve-static@>=1.10.0 <1.11.0", + "resolved": "http://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz" + }, + "type-is": { + "version": "1.6.9", + "from": "type-is@>=1.6.9 <1.7.0", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.9.tgz", + "dependencies": { + "media-typer": { + "version": "0.3.0", + "from": "media-typer@0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "mime-types": { + "version": "2.1.7", + "from": "mime-types@>=2.1.6 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.7.tgz", + "dependencies": { + "mime-db": { + "version": "1.19.0", + "from": "mime-db@>=1.19.0 <1.20.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.19.0.tgz" + } + } + } + } + }, + "utils-merge": { + "version": "1.0.0", + "from": "utils-merge@1.0.0", + "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz" + }, + "vary": { + "version": "1.0.1", + "from": "vary@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz" + } + } + }, + "json": { + "version": "9.0.3", + "from": "json@>=9.0.3 <10.0.0", + "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz" + }, + "morgan": { + "version": "1.6.1", + "from": "morgan@>=1.5.1 <2.0.0", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.6.1.tgz", + "dependencies": { + "basic-auth": { + "version": "1.0.3", + "from": "basic-auth@>=1.0.3 <1.1.0", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-1.0.3.tgz" + }, + "depd": { + "version": "1.0.1", + "from": "depd@>=1.0.1 <1.1.0", + "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.1.tgz" + }, + "on-finished": { + "version": "2.3.0", + "from": "on-finished@>=2.3.0 <2.4.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.1", + "from": "ee-first@1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + } + } + }, + "on-headers": { + "version": "1.0.1", + "from": "on-headers@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz" + } + } + }, + "proxy-middleware": { + "version": "0.15.0", + "from": "proxy-middleware@>=0.15.0 <0.16.0", + "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.15.0.tgz" + }, + "safetydance": { + "version": "0.0.19", + "from": "safetydance@0.0.19", + "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-0.0.19.tgz" + }, + "semver": { + "version": "5.1.0", + "from": "semver@>=5.1.0 <6.0.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz" + }, + "superagent": { + "version": "0.21.0", + "from": "superagent@>=0.21.0 <0.22.0", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz", + "dependencies": { + "component-emitter": { + "version": "1.1.2", + "from": "component-emitter@1.1.2", + "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz" + }, + "cookiejar": { + "version": "2.0.1", + "from": "cookiejar@2.0.1", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz" + }, + "extend": { + "version": "1.2.1", + "from": "extend@>=1.2.1 <1.3.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz" + }, + "form-data": { + "version": "0.1.3", + "from": "form-data@0.1.3", + "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz", + "dependencies": { + "async": { + "version": "0.9.2", + "from": "async@>=0.9.0 <0.10.0", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz" + }, + "combined-stream": { + "version": "0.0.7", + "from": "combined-stream@>=0.0.4 <0.1.0", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", + "dependencies": { + "delayed-stream": { + "version": "0.0.5", + "from": "delayed-stream@0.0.5", + "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz" + } + } + } + } + }, + "formidable": { + "version": "1.0.14", + "from": "formidable@1.0.14", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz" + }, + "methods": { + "version": "1.0.1", + "from": "methods@1.0.1", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz" + }, + "mime": { + "version": "1.2.11", + "from": "mime@1.2.11", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + }, + "qs": { + "version": "1.2.0", + "from": "qs@1.2.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz" + }, + "readable-stream": { + "version": "1.0.27-1", + "from": "readable-stream@1.0.27-1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.1", + "from": "core-util-is@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" + }, + "inherits": { + "version": "2.0.1", + "from": "inherits@>=2.0.1 <2.1.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, + "isarray": { + "version": "0.0.1", + "from": "isarray@0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "string_decoder@>=0.10.0 <0.11.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + } + } + }, + "reduce-component": { + "version": "1.0.1", + "from": "reduce-component@1.0.1", + "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz" + } + } + } + } +} diff --git a/installer/package.json b/installer/package.json new file mode 100644 index 000000000..e24e953a7 --- /dev/null +++ b/installer/package.json @@ -0,0 +1,47 @@ +{ + "name": "installer", + "description": "Cloudron Installer", + "version": "0.0.1", + "private": "true", + "author": { + "name": "Cloudron authors" + }, + "repository": { + "type": "git" + }, + "engines": [ + "node >=4.0.0 <=4.1.1" + ], + "dependencies": { + "async": "^1.5.0", + "body-parser": "^1.12.0", + "connect-lastmile": "0.0.13", + "debug": "^2.1.1", + "express": "^4.11.2", + "json": "^9.0.3", + "morgan": "^1.5.1", + "proxy-middleware": "^0.15.0", + "safetydance": "0.0.19", + "semver": "^5.1.0", + "superagent": "^0.21.0" + }, + "devDependencies": { + "colors": "^1.1.2", + "commander": "^2.8.1", + "expect.js": "^0.3.1", + "istanbul": "^0.3.5", + "lodash": "^3.2.0", + "mocha": "^2.1.0", + "nock": "^0.59.1", + "sleep": "^3.0.0", + "superagent-sync": "^0.2.0", + "supererror": "^0.7.0", + "yesno": "0.0.1" + }, + "scripts": { + "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test", + "precommit": "/bin/true", + "prepush": "npm test", + "postmerge": "/bin/true" + } +} diff --git a/installer/src/certs/.gitignore b/installer/src/certs/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/installer/src/installer.js b/installer/src/installer.js new file mode 100644 index 000000000..eb6a6ac7f --- /dev/null +++ b/installer/src/installer.js @@ -0,0 +1,128 @@ +/* jslint node: true */ + +'use strict'; + +var assert = require('assert'), + child_process = require('child_process'), + debug = require('debug')('installer:installer'), + path = require('path'), + safe = require('safetydance'), + semver = require('semver'), + superagent = require('superagent'), + util = require('util'); + +exports = module.exports = { + InstallerError: InstallerError, + + provision: provision, + retire: retire, + + _ensureVersion: ensureVersion +}; + +var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'), + RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh'), + SUDO = '/usr/bin/sudo'; + +function InstallerError(reason, info) { + Error.call(this); + Error.captureStackTrace(this, this.constructor); + + this.name = this.constructor.name; + this.reason = reason; + this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info); +} +util.inherits(InstallerError, Error); +InstallerError.INTERNAL_ERROR = 1; +InstallerError.ALREADY_PROVISIONED = 2; + +function spawn(tag, cmd, args, callback) { + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof cmd, 'string'); + assert(util.isArray(args)); + assert.strictEqual(typeof callback, 'function'); + + var cp = child_process.spawn(cmd, args, { timeout: 0 }); + cp.stdout.setEncoding('utf8'); + cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); }); + cp.stderr.setEncoding('utf8'); + cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); }); + + cp.on('error', function (error) { + debug('%s : child process errored %s', tag, error.message); + callback(error); + }); + + cp.on('exit', function (code, signal) { + debug('%s : child process exited. code: %d signal: %d', tag, code, signal); + if (signal) return callback(new Error('Exited with signal ' + signal)); + if (code !== 0) return callback(new Error('Exited with code ' + code)); + + callback(null); + }); +} + +function retire(args, callback) { + assert.strictEqual(typeof args, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var pargs = [ RETIRE_CMD ]; + pargs.push('--data', JSON.stringify(args.data)); + + debug('retire: calling with args %j', pargs); + + if (process.env.NODE_ENV === 'test') return callback(null); + + // sudo is required for retire() + spawn('retire', SUDO, pargs, callback); +} + +function ensureVersion(args, callback) { + assert.strictEqual(typeof args, 'object'); + assert.strictEqual(typeof callback, 'function'); + + if (!args.data || !args.data.boxVersionsUrl) return callback(new Error('No boxVersionsUrl specified')); + + if (args.sourceTarballUrl) return callback(null, args); + + superagent.get(args.data.boxVersionsUrl).end(function (error, result) { + if (error && !error.response) return callback(error); + if (result.statusCode !== 200) return callback(new Error(util.format('Bad status: %s %s', result.statusCode, result.text))); + + var versions = safe.JSON.parse(result.text); + + if (!versions || typeof versions !== 'object') return callback(new Error('versions is not in valid format:' + safe.error)); + + var latestVersion = Object.keys(versions).sort(semver.compare).pop(); + debug('ensureVersion: Latest version is %s etag:%s', latestVersion, result.header['etag']); + + if (!versions[latestVersion]) return callback(new Error('No version available')); + if (!versions[latestVersion].sourceTarballUrl) return callback(new Error('No sourceTarballUrl specified')); + + args.sourceTarballUrl = versions[latestVersion].sourceTarballUrl; + args.data.version = latestVersion; + + callback(null, args); + }); +} + +function provision(args, callback) { + assert.strictEqual(typeof args, 'object'); + assert.strictEqual(typeof callback, 'function'); + + if (process.env.NODE_ENV === 'test') return callback(null); + + ensureVersion(args, function (error, result) { + if (error) return callback(error); + + var pargs = [ INSTALLER_CMD ]; + pargs.push('--sourcetarballurl', result.sourceTarballUrl); + pargs.push('--data', JSON.stringify(result.data)); + + debug('provision: calling with args %j', pargs); + + // sudo is required for update() + spawn('provision', SUDO, pargs, callback); + }); +} + diff --git a/installer/src/scripts/installer.sh b/installer/src/scripts/installer.sh new file mode 100755 index 000000000..8f4aa136d --- /dev/null +++ b/installer/src/scripts/installer.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -eu -o pipefail + +readonly BOX_SRC_DIR=/home/yellowtent/box +readonly DATA_DIR=/home/yellowtent/data + +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly json="${script_dir}/../../node_modules/.bin/json" +readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 180" + +readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no") + +# create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@ +(echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh +chmod +x /home/yellowtent/provision.sh + +arg_source_tarball_url="" +arg_data="" + +args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@") +eval set -- "${args}" + +while true; do + case "$1" in + --sourcetarballurl) arg_source_tarball_url="$2";; + --data) arg_data="$2";; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac + + shift 2 +done + +box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX) +echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}" + +while true; do + if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi + echo "Failed to download source tarball, trying again" + sleep 5 +done +(cd "${box_src_tmp_dir}" && npm rebuild) + +if [[ "${is_update}" == "yes" ]]; then + echo "Setting up update splash screen" + "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code + ${BOX_SRC_DIR}/setup/stop.sh # stop the old code +fi + +# switch the codes +rm -rf "${BOX_SRC_DIR}" +mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}" +chown -R yellowtent.yellowtent "${BOX_SRC_DIR}" + +# create a start file for testing. %q escapes args +(echo -e "#!/bin/bash\n"; printf "%q " "${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}") > /home/yellowtent/setup_start.sh +chmod +x /home/yellowtent/setup_start.sh + +echo "Calling box setup script" +"${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}" + diff --git a/installer/src/scripts/retire.sh b/installer/src/scripts/retire.sh new file mode 100755 index 000000000..d93fc7c8e --- /dev/null +++ b/installer/src/scripts/retire.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script is called once at the end of a cloudrons lifetime + +set -eu -o pipefail + +readonly BOX_SRC_DIR=/home/yellowtent/box + +arg_data="" + +args=$(getopt -o "" -l "data:" -n "$0" -- "$@") +eval set -- "${args}" + +while true; do + case "$1" in + --data) arg_data="$2";; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac + + shift 2 +done + +echo "Setting up splash screen" +"${BOX_SRC_DIR}/setup/splashpage.sh" --retire --data "${arg_data}" # show splash +"${BOX_SRC_DIR}/setup/stop.sh" # stop the cloudron code + +systemctl stop docker # stop the apps +systemctl stop cloudron-installer # stop the installer + diff --git a/installer/src/server.js b/installer/src/server.js new file mode 100755 index 000000000..b3628e9c5 --- /dev/null +++ b/installer/src/server.js @@ -0,0 +1,213 @@ +#!/usr/bin/env node + +/* jslint node: true */ + +'use strict'; + +var assert = require('assert'), + async = require('async'), + debug = require('debug')('installer:server'), + express = require('express'), + fs = require('fs'), + http = require('http'), + HttpError = require('connect-lastmile').HttpError, + https = require('https'), + HttpSuccess = require('connect-lastmile').HttpSuccess, + installer = require('./installer.js'), + json = require('body-parser').json, + lastMile = require('connect-lastmile'), + morgan = require('morgan'), + path = require('path'), + superagent = require('superagent'); + +exports = module.exports = { + start: start, + stop: stop +}; + +var PROVISION_CONFIG_FILE = '/root/provision.json'; +var CLOUDRON_CONFIG_FILE = '/home/yellowtent/configs/cloudron.conf'; + +var gHttpsServer = null, // provision server; used for install/restore + gHttpServer = null; // update server; used for updates + +function provisionDigitalOcean(callback) { + if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned + + superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) { + if (error || result.statusCode !== 200) { + console.error('Error getting metadata', error); + return callback(new Error('Error getting metadata')); + } + + var userData = JSON.parse(result.body.user_data); + + installer.provision(userData, callback); + }); +} + +function provisionLocal(callback) { + if (fs.existsSync(CLOUDRON_CONFIG_FILE)) return callback(null); // already provisioned + + if (!fs.existsSync(PROVISION_CONFIG_FILE)) { + console.error('No provisioning data found at %s', PROVISION_CONFIG_FILE); + return callback(new Error('No provisioning data found')); + } + + var userData = require(PROVISION_CONFIG_FILE); + + installer.provision(userData, callback); +} + +function update(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (!req.body.sourceTarballUrl || typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided')); + if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided')); + + debug('provision: received from box %j', req.body); + + installer.provision(req.body, function (error) { + if (error) console.error(error); + }); + + next(new HttpSuccess(202, { })); +} + +function retire(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided')); + + if (typeof req.body.data.tlsCert !== 'string') console.error('No TLS cert provided'); + if (typeof req.body.data.tlsKey !== 'string') console.error('No TLS key provided'); + + debug('retire: received from appstore %j', req.body); + + installer.retire(req.body, function (error) { + if (error) console.error(error); + }); + + next(new HttpSuccess(202, {})); +} + +function startUpdateServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Starting update server'); + + var app = express(); + + var router = new express.Router(); + + if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false })); + + app.use(json({ strict: true })) + .use(router) + .use(lastMile()); + + router.post('/api/v1/installer/update', update); + + gHttpServer = http.createServer(app); + gHttpServer.on('error', console.error); + + gHttpServer.listen(2020, '127.0.0.1', callback); +} + +function startProvisionServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Starting provision server'); + + var app = express(); + + var router = new express.Router(); + + if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false })); + + app.use(json({ strict: true })) + .use(router) + .use(lastMile()); + + router.post('/api/v1/installer/retire', retire); + + var caPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? 'test/certs' : 'certs'); + var certPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? 'test/certs' : 'certs'); + + var options = { + key: fs.readFileSync(path.join(certPath, 'server.key')), + cert: fs.readFileSync(path.join(certPath, 'server.crt')), + ca: fs.readFileSync(path.join(caPath, 'ca.crt')), + + // request cert from client and only allow from our CA + requestCert: true, + rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0' // this is set in the tests + }; + + gHttpsServer = https.createServer(options, app); + gHttpsServer.on('error', console.error); + + gHttpsServer.listen(process.env.NODE_ENV === 'test' ? 4443 : 886, '0.0.0.0', callback); +} + +function stopProvisionServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Stopping provision server'); + + if (!gHttpsServer) return callback(null); + + gHttpsServer.close(callback); + gHttpsServer = null; +} + +function stopUpdateServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Stopping update server'); + + if (!gHttpServer) return callback(null); + + gHttpServer.close(callback); + gHttpServer = null; +} + +function start(callback) { + assert.strictEqual(typeof callback, 'function'); + + var actions; + + if (process.env.PROVISION === 'local') { + debug('Starting Installer in selfhost mode'); + + actions = [ + startUpdateServer, + provisionLocal + ]; + } else { // current fallback, should be 'digitalocean' eventually, see initializeBaseUbuntuImage.sh + debug('Starting Installer in managed mode'); + + actions = [ + startUpdateServer, + startProvisionServer, + provisionDigitalOcean + ]; + } + + async.series(actions, callback); +} + +function stop(callback) { + assert.strictEqual(typeof callback, 'function'); + + async.series([ + stopUpdateServer, + stopProvisionServer + ], callback); +} + +if (require.main === module) { + start(function (error) { + if (error) console.error(error); + }); +} diff --git a/installer/src/test/certs/ca.crt b/installer/src/test/certs/ca.crt new file mode 100644 index 000000000..e053e9bef --- /dev/null +++ b/installer/src/test/certs/ca.crt @@ -0,0 +1,24 @@ +-----BEGIN CERTIFICATE----- +MIID9zCCAt+gAwIBAgIJAMPL81PAySGAMA0GCSqGSIb3DQEBBQUAMFoxCzAJBgNV +BAYTAlVTMQswCQYDVQQIEwJDQTELMAkGA1UEBxMCU0MxFTATBgNVBAoTDENsb3Vk +cm9uIEluYzEaMBgGA1UEAxMRSW5zdGFsbCBTZXJ2ZXIgQ0EwHhcNMTUwMTE2MDEy +NDM2WhcNMTYwMTE2MDEyNDM2WjBaMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0Ex +CzAJBgNVBAcTAlNDMRUwEwYDVQQKEwxDbG91ZHJvbiBJbmMxGjAYBgNVBAMTEUlu +c3RhbGwgU2VydmVyIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +31TkOEC3JXtieHiZgM5qWw771rV2JEDKs1C68+n/OmKrp3zAQV08A+w/KVurn1P9 +gZlYF+CBRVZDV8lYbWzc6PgMPWEDHHV72FS5Kq6ZyikB+r5OQJ8qU61y840h6ZCD +MEYr6N9qXm9wSApJBQ/key/pg7+95B2CFYRrg5NVstIYqpJ1lyxCMFTrjYAmteOB +Bi/4GPApu9Tj0ifTMbZFGTPtWm/yhCZ6Anm6w+ok9tDMpPC6kRgUJ3B4HY75D9dV +aWSls9jdZw4JU1jIFlAdUjhGEEmHWOzAD8vBjvuBqcf9NQwvieWG5tDYfZ6DYRC2 +/aG1C5UWhFLDv2/F+56k3wIDAQABo4G/MIG8MB0GA1UdDgQWBBQ088hd2sIIqVtw +xJeAkCORdclFRjCBjAYDVR0jBIGEMIGBgBQ088hd2sIIqVtwxJeAkCORdclFRqFe +pFwwWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMQswCQYDVQQHEwJTQzEVMBMG +A1UEChMMQ2xvdWRyb24gSW5jMRowGAYDVQQDExFJbnN0YWxsIFNlcnZlciBDQYIJ +AMPL81PAySGAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAJcW+Wmz +/o0JBC2WsMjUjxVrzOiu9bdKQ1yn83Zcv74zEfmWfJotVOK1oKsTyOZfTvvWrpLc +GXXhh4oXWsNnFII3uJyZIY3v/DoE0pa7TCZhLYFbL2kEaC5rTwe/+VScHy5ROOiu ++gnzOU3MyrcMTT0v4qcT0NlkIptRdvIYNpqfXO6vG9sMp4C/NwWhl/IfHkIAv0eH +l3HTr8wxgldCjxbnJgYkyUcWAmLi2YEXKCEPWmsfqp3Z+Ng1M+A9OKjJLHWowl9X +4arvn6WaUbZjRxxjvK199If1R6KWwD6YQ9cKH4Ex4/hhIqg5I3MQFu+pOq/b0XH/ +9I10o6FVU7vcFkQ= +-----END CERTIFICATE----- diff --git a/installer/src/test/certs/server.crt b/installer/src/test/certs/server.crt new file mode 100644 index 000000000..7480dde1a --- /dev/null +++ b/installer/src/test/certs/server.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDMDCCAhgCCQCDr1HQJBr1izANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNDMREwDwYDVQQKDAhDbG91ZHJvbjEe +MBwGA1UEAwwVaW5zdGFsbGVyLmNsb3Vkcm9uLmlvMB4XDTE1MTExNjIzMTcwMloX +DTE2MTExNTIzMTcwMlowWjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQswCQYD +VQQHDAJTQzERMA8GA1UECgwIQ2xvdWRyb24xHjAcBgNVBAMMFWluc3RhbGxlci5j +bG91ZHJvbi5pbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK0suQX7 +hKBhYsSH0msnEPVbRDIotYbtVDav/v7Sb/fRU7qVoL31tj2iZRDJRJ27uRM3J4ye +6hgJAAwQGtfXrcVZY3SOAlGXsFZF0wgBCw0pGtgF3HA1BcwbCwAd06J6w3lKActA +DMEUio/jRXpYELUU2Nzopq0MsMyyBSBkNC18i0HUB8vkF8yQvb1OpbcxERbpf3D5 +zjeFf5kIE/k8lwBz1vMF0uAA2GfcXxs3dyDaxVteWeevVYZzAoY9EcUyBWX7OQnx +aUygl3OywN+xOJKXKCQpckzDvr9Vp1sKItoMMy5y81SyNhZIMBYGGG+oNp/wSgQf +Cht+LupI+bXoYrMCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAgPHZx52qYuEUdzVO +t/+VXO7dxJkONYU8sjTYIfJme8ZZd7beZBMUni5s2gvv6i5HFyJ2Ol88sv8hAaI/ +6Vmbszml+5tLyPK8Gygk62l6OcKDwU/yazTxxCApulNy1SV34kzruXUMZ28ybcqA +XJywMMx4RDmSIBXPdDCeaOgYwI7Wk56obJ8sa2+Z6100GNoX+qBSOsWMMJW+ohnp +eQWHkTOJzU4hIMfZCbW0cF5Xn/35xEh0xxaH7XWglJLM9neBPba+Ydz7567mN9co +vgv2dE5ZOKSjG63CtUvv819dvbWVKq8jiMCqPGRcr1iSeqbC02tnx0W762980uSx +QfOgAw== +-----END CERTIFICATE----- diff --git a/installer/src/test/certs/server.key b/installer/src/test/certs/server.key new file mode 100644 index 000000000..0a732c4d3 --- /dev/null +++ b/installer/src/test/certs/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEArSy5BfuEoGFixIfSaycQ9VtEMii1hu1UNq/+/tJv99FTupWg +vfW2PaJlEMlEnbu5EzcnjJ7qGAkADBAa19etxVljdI4CUZewVkXTCAELDSka2AXc +cDUFzBsLAB3TonrDeUoBy0AMwRSKj+NFelgQtRTY3OimrQywzLIFIGQ0LXyLQdQH +y+QXzJC9vU6ltzERFul/cPnON4V/mQgT+TyXAHPW8wXS4ADYZ9xfGzd3INrFW15Z +569VhnMChj0RxTIFZfs5CfFpTKCXc7LA37E4kpcoJClyTMO+v1WnWwoi2gwzLnLz +VLI2FkgwFgYYb6g2n/BKBB8KG34u6kj5tehiswIDAQABAoIBAGNAQ5bbLYsh5ZKP +6ZhCHqUQtsgsrsVzFhX1zqbLgyK8VUmV4jedMOKoRVZWlD32zj7mGIOuvKoj1mQT +gt78HPsDnU266jdLQeRgRm/K8UOMsHbo/QtOSFFPmoFpltcDly7XrKmJvwWWOUf4 +UOSqvoCaPyR1Lrn1kQrwaKHE7Ga4jfyOrIq9JI7y/ih+Y7D8xcMnyLAsjyVkSAtr ++XrGNHcx3yPuBmjaOglzeb6Ksdpt4ETElrvH3ByT5EV2zUVr9Txv+m8xSVBZfea9 +aE7lWSQoOUz+e6RhIX3Df/QfR6KkDblAwEF9Se98DWcz46Y34oc2E0lSoJYpoPxP +vbRlfDkCgYEA3nAc8kDRkbQObSfnVjpijBSP5hfr3jX+XTbxK7Y3aTMViY+87iWK +bLNuX+2JRCmRjk0wy2YXnJQV3sU/EO5gLhOz9060MIHgFISq4KRgPorN/EFWryOe +mDzhPIuhZLMetv0ajS3Z5IxIAs+FLu7Yx9em80q540UA3kXsFWe2lpUCgYEAx03E +kk5zLirVFtoyP/yAES+KVppqBweCUA5vVxB8H26oIhi8G8kT4b77x6wXxQzdsA4H +a4ou3ZBZVK41PREgG1MWgzpbwk49T1FX6TLtvdhr/9QhYC+RIynynA/pA36LSKT5 +pvWegYB4+9jaPrQ5L1zcrLF2XlTsgpuC43kXKicCgYA0dXxeJatHEY/VbnPAgkR7 +hN3rBfk6jsFOeoamKHMo/EM4Dg4gm/npaOe+9+ZHjQYm6U14qrsm0kXWI+6br5w/ +QaZPzN/yEK8oJ6GlGR8ZoOKzezVWWLAudy0neka12QiFX2vDn+yjWfIht49RYkL9 +3n4hIp50WvG5egQTiEIngQKBgCn9yJzKypm/jIX0EwJIQPNeANeeURiKDHqxj+PY +JU66EdKdQ4TXKMk3Y/T93UQ3Ib4mNooB4z3rW+brjWwAX7NiHiwn741QzroXeV44 +zL5jCt4r45xQaVPvUp5u+7kwwEfd+nui5HKEjvkBB3qOnj3MYvI/saDOY8Zg3YLv +0GGhAoGANBwFcDgwP9KDt0NxKXhe3rlSUyfGSSUF89hZPrLDCiaGFURD/w4j3EGr +Ui9Rcwm2ymqlFzTO4JYKy1/pRCWA7GDfslICJPOPG3Wytsjog0WymQuMjYC2tL/+ +RwD0qG0/aBGE4PbigPRoJ/7BGZLKtdy99P0wyFC3o6OBoAl3Zqo= +-----END RSA PRIVATE KEY----- diff --git a/installer/src/test/installer-test.js b/installer/src/test/installer-test.js new file mode 100644 index 000000000..cb7b26731 --- /dev/null +++ b/installer/src/test/installer-test.js @@ -0,0 +1,219 @@ +/* jslint node:true */ +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +'use strict'; + +var expect = require('expect.js'), + fs = require('fs'), + path = require('path'), + nock = require('nock'), + os = require('os'), + request = require('superagent'), + server = require('../server.js'), + installer = require('../installer.js'), + _ = require('lodash'); + +var EXTERNAL_SERVER_URL = 'https://localhost:4443'; +var INTERNAL_SERVER_URL = 'http://localhost:2020'; +var APPSERVER_ORIGIN = 'http://appserver'; +var FQDN = os.hostname(); + +describe('Server', function () { + this.timeout(5000); + + before(function (done) { + var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string + var scope = nock('http://169.254.169.254') + .persist() + .get('/metadata/v1.json') + .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' }); + done(); + }); + + after(function (done) { + nock.cleanAll(); + done(); + }); + + describe('starts and stop', function () { + it('starts', function (done) { + server.start(done); + }); + + it('stops', function (done) { + server.stop(done); + }); + }); + + describe('update (internal server)', function () { + before(function (done) { + server.start(done); + }); + after(function (done) { + server.stop(done); + }); + + it('does not respond to provision', function (done) { + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(404); + done(); + }); + }); + + it('does not respond to restore', function (done) { + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(404); + done(); + }); + }); + + var data = { + sourceTarballUrl: "https://foo.tar.gz", + + data: { + token: 'sometoken', + apiServerOrigin: APPSERVER_ORIGIN, + webServerOrigin: 'https://somethingelse.com', + fqdn: 'www.something.com', + tlsKey: 'key', + tlsCert: 'cert', + boxVersionsUrl: 'https://versions.json', + version: '0.1' + } + }; + + Object.keys(data).forEach(function (key) { + it('fails due to missing ' + key, function (done) { + var dataCopy = _.merge({ }, data); + delete dataCopy[key]; + + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + }); + + it('succeeds', function (done) { + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(202); + done(); + }); + }); + }); + + describe('retire', function () { + var data = { + data: { + tlsKey: 'key', + tlsCert: 'cert' + } + }; + + before(function (done) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead + server.start(done); + }); + + after(function (done) { + server.stop(done); + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + }); + + Object.keys(data).forEach(function (key) { + it('fails due to missing ' + key, function (done) { + var dataCopy = _.merge({ }, data); + delete dataCopy[key]; + + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(dataCopy).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + }); + + it('succeeds', function (done) { + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(data).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(202); + done(); + }); + }); + }); + + describe('ensureVersion', function () { + before(function () { + process.env.NODE_ENV = undefined; + }); + + after(function () { + process.env.NODE_ENV = 'test'; + }); + + it ('fails without data', function (done) { + installer._ensureVersion({}, function (error) { + expect(error).to.be.an(Error); + done(); + }); + }); + + it ('fails without boxVersionsUrl', function (done) { + installer._ensureVersion({ data: {}}, function (error) { + expect(error).to.be.an(Error); + done(); + }); + }); + + it ('succeeds with sourceTarballUrl', function (done) { + var data = { + sourceTarballUrl: 'sometarballurl', + data: { + boxVersionsUrl: 'http://foobar/versions.json' + } + }; + + installer._ensureVersion(data, function (error, result) { + expect(error).to.equal(null); + expect(result).to.eql(data); + done(); + }); + }); + + it ('succeeds without sourceTarballUrl', function (done) { + var versions = { + '0.1.0': { + sourceTarballUrl: 'sometarballurl1' + }, + '0.2.0': { + sourceTarballUrl: 'sometarballurl2' + } + }; + + var scope = nock('http://foobar') + .get('/versions.json') + .reply(200, JSON.stringify(versions), { 'Content-Type': 'application/json' }); + + var data = { + data: { + boxVersionsUrl: 'http://foobar/versions.json' + } + }; + + installer._ensureVersion(data, function (error, result) { + expect(error).to.equal(null); + expect(result.sourceTarballUrl).to.equal(versions['0.2.0'].sourceTarballUrl); + expect(result.data.boxVersionsUrl).to.equal(data.data.boxVersionsUrl); + done(); + }); + }); + }); +}); + diff --git a/installer/systemd/box-setup.sh b/installer/systemd/box-setup.sh new file mode 100755 index 000000000..d827e316a --- /dev/null +++ b/installer/systemd/box-setup.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -eu -o pipefail + +readonly USER_HOME="/home/yellowtent" +readonly APPS_SWAP_FILE="/apps.swap" +readonly BACKUP_SWAP_FILE="/backup.swap" # used when doing app backups +readonly USER_DATA_FILE="/root/user_data.img" +readonly USER_DATA_DIR="/home/yellowtent/data" + +# detect device +if [[ -b "/dev/vda1" ]]; then + disk_device="/dev/vda1" +fi + +if [[ -b "/dev/xvda1" ]]; then + disk_device="/dev/xvda1" +fi + +# all sizes are in mb +readonly physical_memory=$(free -m | awk '/Mem:/ { print $2 }') +readonly swap_size="${physical_memory}" +readonly app_count=$((${physical_memory} / 200)) # estimated app count +readonly disk_size_gb=$(fdisk -l ${disk_device} | grep "Disk ${disk_device}" | awk '{ print $3 }') +readonly disk_size=$((disk_size_gb * 1024)) +readonly backup_swap_size=1024 +# readonly system_size=5120 # 5 gigs for system libs, installer, box code and tmp +readonly system_size=10240 # 10 gigs for system libs, apps images, installer, box code and tmp +readonly ext4_reserved=$((disk_size * 5 / 100)) # this can be changes using tune2fs -m percent /dev/vda1 + +echo "Disk device: ${disk_device}" +echo "Physical memory: ${physical_memory}" +echo "Estimated app count: ${app_count}" +echo "Disk size: ${disk_size}" + +# Allocate two sets of swap files - one for general app usage and another for backup +# The backup swap is setup for swap on the fly by the backup scripts +if [[ ! -f "${APPS_SWAP_FILE}" ]]; then + echo "Creating Apps swap file of size ${swap_size}M" + fallocate -l "${swap_size}m" "${APPS_SWAP_FILE}" + chmod 600 "${APPS_SWAP_FILE}" + mkswap "${APPS_SWAP_FILE}" + swapon "${APPS_SWAP_FILE}" + echo "${APPS_SWAP_FILE} none swap sw 0 0" >> /etc/fstab +else + echo "Apps Swap file already exists" +fi + +if [[ ! -f "${BACKUP_SWAP_FILE}" ]]; then + echo "Creating Backup swap file of size ${backup_swap_size}M" + fallocate -l "${backup_swap_size}m" "${BACKUP_SWAP_FILE}" + chmod 600 "${BACKUP_SWAP_FILE}" + mkswap "${BACKUP_SWAP_FILE}" +else + echo "Backups Swap file already exists" +fi + +echo "Resizing data volume" +home_data_size=$((disk_size - system_size - swap_size - backup_swap_size - ext4_reserved)) +echo "Resizing up btrfs user data to size ${home_data_size}M" +umount "${USER_DATA_DIR}" +fallocate -l "${home_data_size}m" "${USER_DATA_FILE}" # does not overwrite existing data +mount "${USER_DATA_FILE}" +btrfs filesystem resize max "${USER_DATA_DIR}" +