commit d3fb244cef50240bfe33c46b153a68cb7c4f1da0 Author: Girish Ramakrishnan Date: Tue Aug 4 16:29:49 2015 -0700 list ldap as 0.0.25 change diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..ef7233d0e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# following files are skipped when exporting using git archive +/release export-ignore +/images export-ignore +/admin export-ignore +test export-ignore +.gitattributes export-ignore +.gitignore export-ignore + diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..767e764bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +src/certs/server.key + diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 000000000..ad6d169fb --- /dev/null +++ b/.jshintrc @@ -0,0 +1,7 @@ +{ + "node": true, + "browser": true, + "unused": true, + "globalstrict": true, + "predef": [ "angular", "$" ] +} diff --git a/admin/admin b/admin/admin new file mode 100755 index 000000000..7e79bb534 --- /dev/null +++ b/admin/admin @@ -0,0 +1,340 @@ +#!/usr/bin/env node + +'use strict'; + +var assert = require('assert'), + async = require('async'), + crypto = require('crypto'), + execSync = require('child_process').execSync, + fs = require('fs'), + https = require('https'), + os = require('os'), + path = require('path'), + program = require('commander'), + readlineSync = require('readline-sync'), + spawn = require('child_process').spawn, + SshClient = require('ssh2').Client, + superagent = require('superagent'), + util = require('util'); + +require('colors'); + +var SSH = 'root@%s -tt -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i %s'; +var sshKeyPath = path.join(process.env.HOME, '/.ssh/id_rsa_yellowtent'); + +if (!process.env['DIGITAL_OCEAN_TOKEN_DEV']) exit('Missing env variable DIGITAL_OCEAN_TOKEN_DEV'); +if (!process.env['DIGITAL_OCEAN_TOKEN_STAGING']) exit('Missing env variable DIGITAL_OCEAN_TOKEN_STAGING'); + +if (!fs.existsSync(sshKeyPath)) exit('Unable to find ssh key path. Searching for ' + sshKeyPath); + +// Allow self signed certs! +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; + +function exit(error) { + if (error) console.log(error); + process.exit(error ? 1 : 0); +} + +function getDroplets(token, callback) { + assert.strictEqual(typeof token, 'string'); + assert.strictEqual(typeof callback, 'function'); + + var droplets = []; + var nextPage = null; + + async.doWhilst(function (callback) { + var url = nextPage ? nextPage : 'https://api.digitalocean.com/v2/droplets'; + + superagent.get(url).set('Authorization', 'Bearer ' + token).end(function (error, result) { + if (error) return callback(error.message); + if (result.statusCode === 403) return callback('Invalid Digitalocean credentials'); + if (result.statusCode !== 200) return callback(util.format('Unable to get droplet list. %s - %s', result.statusCode, result.text)); + + nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null; + droplets = droplets.concat(result.body.droplets); + + callback(null); + }); + }, function () { return !!nextPage; }, function (error) { + if (error) return callback(error); + callback(null, droplets); + }); +} + +function selectCloudron(action) { + assert.strictEqual(typeof action, 'function'); + + var dropletsDev = []; + var dropletsStaging = []; + var dropletsProd = []; + + console.log('Getting droplet lists from dev and staging...'); + + getDroplets(process.env['DIGITAL_OCEAN_TOKEN_DEV'], function (error, result) { + if (error) exit(error); + + dropletsDev = result; + + getDroplets(process.env['DIGITAL_OCEAN_TOKEN_STAGING'], function (error, result) { + if (error) exit(error); + + dropletsStaging = result; + + getDroplets(process.env['DIGITAL_OCEAN_TOKEN_PROD'], function (error, result) { + if (error) exit(error); + + dropletsProd = result; + + console.log(); + console.log('Available Droplets on dev:'.bold); + dropletsDev.forEach(function (droplet, index) { + console.log('\t(%s)\t%s %s', index, droplet.name.cyan, droplet.networks.v4[0].ip_address); + }); + + console.log(); + console.log('Available Droplets on staging:'.bold); + dropletsStaging.forEach(function (droplet, index) { + console.log('\t(%s)\t%s %s', dropletsDev.length + index, droplet.name.cyan, droplet.networks.v4[0].ip_address); + }); + + console.log(); + console.log('Available Droplets on prod:'.bold); + dropletsProd.forEach(function (droplet, index) { + console.log('\t(%s)\t%s %s', dropletsDev.length + dropletsStaging.length + index, droplet.name.cyan, droplet.networks.v4[0].ip_address); + }); + + console.log(); + + var droplets = dropletsDev.concat(dropletsStaging).concat(dropletsProd); + + var index = -1; + while (true) { + index = parseInt(readlineSync.question('Choose cloudron [0-' + (droplets.length-1) + ']: ', {})); + if (isNaN(index) || index < 0 || index > droplets.length-1) console.log('Invalid selection'.red); + else break; + } + + action(droplets[index].networks.v4[0].ip_address); + }); + }); + }); +} + +function loginToCloudron(ip) { + assert.strictEqual(typeof ip, 'string'); + + console.log('Ssh into %s'.bold, ip.cyan); + + var ssh = spawn('ssh', util.format(SSH, ip, sshKeyPath).split(' ')); + ssh.on('exit', exit); + ssh.on('error', exit); + + process.stdin.setEncoding('utf8'); + process.stdin.setRawMode(true); + + process.stdin.pipe(ssh.stdin); + ssh.stdout.pipe(process.stdout); + ssh.stderr.pipe(process.stderr); + + process.stdin.resume(); + +} + +function logsFromCloudron(ip, fileName, tail) { + assert.strictEqual(typeof ip, 'string'); + assert.strictEqual(typeof fileName, 'string'); + assert.strictEqual(typeof tail, 'boolean'); + + console.log('Fetching logs from'.bold, ip.cyan); + + var options = { + hostname: ip, + port: 886, + path: util.format('/api/v1/installer/logs?filename=%s&tail=%s', fileName, tail), + method: 'GET', + key: fs.readFileSync(path.join(__dirname, '../../keys/installer/server.key')), + cert: fs.readFileSync(path.join(__dirname, '../../keys/installer/server.crt')), + ca: fs.readFileSync(path.join(__dirname, '../../keys/installer_ca/ca.crt')), + rejectUnauthorized: false + }; + + var req = https.request(options, function (res) { + res.setEncoding('utf8'); + res.on('data', function (chunk) { + process.stdout.write(chunk); + }); + }); + + req.on('error', function (error) { + exit(error); + }); + + req.end(); +} + +function triggerBackup(ip) { + assert.strictEqual(typeof ip, 'string'); + + console.log('Trigger backup on %s'.bold, ip.cyan); + + var options = { + hostname: ip, + port: 886, + path: '/api/v1/installer/backup', + method: 'POST', + key: fs.readFileSync(path.join(__dirname, '../../keys/installer/server.key')), + cert: fs.readFileSync(path.join(__dirname, '../../keys/installer/server.crt')), + ca: fs.readFileSync(path.join(__dirname, '../../keys/installer_ca/ca.crt')), + rejectUnauthorized: false + }; + + var req = https.request(options, function (res) { + res.setEncoding('utf8'); + res.on('data', function (chunk) { + process.stdout.write(chunk); + }); + }); + + req.on('error', function (error) { + exit(error); + }); + + req.end(); +} + +function sshExec(ip, cmds) { + var privateKey = path.join(process.env.HOME, '.ssh/id_rsa_yellowtent'); + if (!fs.existsSync(privateKey)) exit('cannot find private key'); + + var sshClient = new SshClient(); + sshClient.connect({ + host: ip, + port: 22, + username: 'root', + privateKey: fs.readFileSync(privateKey) + }); + sshClient.on('ready', function () { + console.log('connected'); + + async.eachSeries(cmds, function (cmd, iteratorDone) { + console.log(cmd.cmd.yellow); + + sshClient.exec(cmd.cmd, function(err, stream) { + if (err) exit(err.message); + + if (cmd.stdin) cmd.stdin.pipe(stream); + stream.pipe(process.stdout); + stream.on('close', function () { + iteratorDone(); + }); + }); + }, function seriesDone(error) { + if (error) exit(error.message); + + console.log('Done patching'.green); + sshClient.end(); + }); + }); + sshClient.on('error', function (error) { + exit(error.message); + }); + sshClient.on('exit', function (exitCode) { + console.log('exit'); + process.exit(exitCode); + }); +} + +function hotfixCloudron(ip, code) { + var CMDS = [ + { cmd: 'supervisorctl stop all' }, + { cmd: 'rm -rf /home/yellowtent/box/* /home/yellowtent/box/.*' }, + { cmd: 'tar zxf - -C /home/yellowtent/box', stdin: fs.createReadStream(code) }, + { cmd: 'cd /home/yellowtent/box && npm rebuild' }, + { cmd: 'chown -R yellowtent.yellowtent /home/yellowtent/box' }, + { cmd: 'sed -e "s/restoreUrl/_restoreUrl/" -i /home/yellowtent/setup_start.sh' }, // do not restore + { cmd: '/home/yellowtent/setup_start.sh' } // ensure db-migrate runs as well + ]; + + sshExec(ip, CMDS); +} + +function hotfix(options) { + var code; + + if (!options.code) { + var answer = readlineSync.question('Create a tarball from repo (y/n)? '); + if (answer !== 'y') return exit(); + code = os.tmpdir() + '/boxtarball.tar.gz'; + execSync(path.join(__dirname, '../images/createBoxTarball --output ' + code + ' --no-upload'), { stdio: [ null, process.stdout, process.stderr ] }); + } else { + code = options.code; + } + + if (!options.ip) { + selectCloudron(function (ip) { hotfixCloudron(ip, code); }); + } else { + hotfixCloudron(options.ip, code); + } +} + +function login(options) { + if (!options.ip) selectCloudron(loginToCloudron); + else loginToCloudron(options.ip); +} + +function logs(options) { + var fileName = '/var/log/supervisor/box.log'; + + if (options.installer) fileName = '/var/log/cloudron/installserver.log'; + if (options.nginxAccess) fileName = '/var/log/nginx/access.log'; + if (options.nginxError) fileName = '/var/log/nginx/error.log'; + + if (!options.ip) selectCloudron(function (ip) { logsFromCloudron(ip, fileName, !!options.tail); }); + else logsFromCloudron(options.ip, fileName, !!options.tail); +} + +function backup(options) { + if (!options.ip) selectCloudron(triggerBackup); + else triggerBackup(options.ip); +} + +// entry point +program.version('0.1.0'); + +program.command('login') + .description('Login to cloudron') + .option('--ip ', 'Cloudron IP') + .action(login); + +program.command('logs') + .description('Fetch logs by filename') + .option('--ip ', 'Cloudron IP') + .option('-f, --tail', 'tail the logs') + .option('--installer', 'installer logs') + .option('--nginx-error', 'nginx error logs') + .option('--nginx-access', 'nginx access logs') + .option('--box', 'box logs [default]') + .action(logs); + +program.command('hotfix') + .description('Hotfix a cloudron') + .option('--ip ', 'Cloudron IP') + .option('--code ', 'Code tarball') + .action(hotfix); + +program.command('backup') + .description('Backup a cloudron') + .option('--ip ', 'Cloudron IP') + .action(backup); + +program.parse(process.argv); + +if (!process.argv.slice(2).length) { + program.outputHelp(); +} else { // https://github.com/tj/commander.js/issues/338 + var knownCommand = program.commands.some(function (command) { return command._name === process.argv[2]; }); + if (!knownCommand) { + console.error('Unknown command: ' + process.argv[2]); + process.exit(1); + } +} diff --git a/admin/cloudronLogin b/admin/cloudronLogin new file mode 100755 index 000000000..4e62de8c7 --- /dev/null +++ b/admin/cloudronLogin @@ -0,0 +1,17 @@ +#!/bin/bash + +set -eu -o pipefail + +readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent" + +if [[ "$#" != "1" ]]; then + echo "Missing cloudron IP argument"; + exit 1; +fi + +if [[ ! -f "${ssh_keys}" ]]; then + echo "yellowtent ssh key is missing" + exit 1 +fi + +ssh root@$1 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "${ssh_keys}" diff --git a/images/createBoxTarball b/images/createBoxTarball new file mode 100755 index 000000000..cbbfeafce --- /dev/null +++ b/images/createBoxTarball @@ -0,0 +1,118 @@ +#!/bin/bash + +set -eu + +assertNotEmpty() { + : "${!1:? "$1 is not set."}" +} + +# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt +# brew install gnu-getopt to get the GNU getopt on OS X +[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt" +readonly GNU_GETOPT + +args=$(${GNU_GETOPT} -o "" -l "revision:,output:,publish,no-upload" -n "$0" -- "$@") +eval set -- "${args}" + +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly box_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../box" && pwd)" + +delete_bundle="yes" +commitish="HEAD" +publish="no" +upload="yes" +bundle_file="" + +while true; do + case "$1" in + --revision) commitish="$2"; shift 2;; + --output) bundle_file="$2"; delete_bundle="no"; shift 2;; + --no-upload) upload="no"; shift;; + --publish) publish="yes"; shift;; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac +done + +if [[ "${upload}" == "no" && "${publish}" == "yes" ]]; then + echo "Cannot publish without uploading" + exit 1 +fi + +readonly TMPDIR=${TMPDIR:-/tmp} # why is this not set on mint? + +assertNotEmpty AWS_DEV_ACCESS_KEY +assertNotEmpty AWS_DEV_SECRET_KEY + +if ! $(cd "${box_dir}" && git diff --exit-code >/dev/null); then + echo "You have local changes, stash or commit them to proceed" + exit 1 +fi + +version=$(cd "${box_dir}" && git rev-parse "${commitish}") +bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR) +[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${version}.tar.gz" + +chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group +echo "Checking out code [${version}] into ${bundle_dir}" +(cd "${box_dir}" && git archive --format=tar ${version} | (cd "${bundle_dir}" && tar xf -)) + +if diff "${TMPDIR}/boxtarball.cache/package.json.all" "${bundle_dir}/package.json" >/dev/null 2>&1; then + echo "Reusing dev modules from cache" + cp -r "${TMPDIR}/boxtarball.cache/node_modules-all/." "${bundle_dir}/node_modules" +else + echo "Installing modules with dev dependencies" + (cd "${bundle_dir}" && npm install) + + echo "Caching dev dependencies" + mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-all" + rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-all/" + cp "${bundle_dir}/package.json" "${TMPDIR}/boxtarball.cache/package.json.all" +fi + +echo "Building webadmin assets" +(cd "${bundle_dir}" && gulp) + +echo "Remove intermediate files required at build-time only" +rm -rf "${bundle_dir}/node_modules/" +rm -rf "${bundle_dir}/webadmin/src" +rm -rf "${bundle_dir}/gulpfile.js" + +if diff "${TMPDIR}/boxtarball.cache/package.json.prod" "${bundle_dir}/package.json" >/dev/null 2>&1; then + echo "Reusing prod modules from cache" + cp -r "${TMPDIR}/boxtarball.cache/node_modules-prod/." "${bundle_dir}/node_modules" +else + echo "Installing modules for production" + (cd "${bundle_dir}" && npm install --production) + + echo "Caching prod dependencies" + mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-prod" + rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-prod/" + cp "${bundle_dir}/package.json" "${TMPDIR}/boxtarball.cache/package.json.prod" +fi + +echo "Create final tarball" +(cd "${bundle_dir}" && tar czf "${bundle_file}" .) +echo "Cleaning up ${bundle_dir}" +rm -rf "${bundle_dir}" + +if [[ "${upload}" == "yes" ]]; then + echo "Uploading bundle to S3" + # That special header is needed to allow access with singed urls created with different aws credentials than the ones the file got uploaded + s3cmd --ssl --add-header=x-amz-acl:authenticated-read --access_key="${AWS_DEV_ACCESS_KEY}" --secret_key="${AWS_DEV_SECRET_KEY}" --no-mime-magic put "${bundle_file}" "s3://dev-cloudron-releases/box-${version}.tar.gz" + + versions_file_url="https://dev-cloudron-releases.s3.amazonaws.com/box-${version}.tar.gz" + echo "The URL for the versions file is: ${versions_file_url}" + + if [[ "${publish}" == "yes" ]]; then + echo "Publishing to dev" + ${script_dir}/release/release create --env dev --code "${versions_file_url}" + fi +fi + +if [[ "${delete_bundle}" == "no" ]]; then + echo "Tarball preserved at ${bundle_file}" +else + rm "${bundle_file}" +fi + diff --git a/images/createDigitalOceanImage.sh b/images/createDigitalOceanImage.sh new file mode 100755 index 000000000..5755a4c70 --- /dev/null +++ b/images/createDigitalOceanImage.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +set -eu -o pipefail + +assertNotEmpty() { + : "${!1:? "$1 is not set."}" +} + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly INSTALLER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" +readonly JSON="${INSTALLER_DIR}/node_modules/.bin/json" +readonly ssh_keys="${HOME}/.ssh/id_rsa_yellowtent" + +installer_revision=$(git rev-parse HEAD) +box_size="512mb" +image_regions=(sfo1 ams3) +box_name="" +droplet_id="" +droplet_ip="" +destroy_droplet="yes" +deploy_env="dev" + +# Only GNU getopt supports long options. OS X comes bundled with the BSD getopt +# brew install gnu-getopt to get the GNU getopt on OS X +[[ $(uname -s) == "Darwin" ]] && GNU_GETOPT="/usr/local/opt/gnu-getopt/bin/getopt" || GNU_GETOPT="getopt" +readonly GNU_GETOPT + +args=$(${GNU_GETOPT} -o "" -l "revision:,regions:,size:,box:,no-destroy,env:" -n "$0" -- "$@") +eval set -- "${args}" + +while true; do + case "$1" in + --env) deploy_env="$2"; shift 2;; + --revision) installer_revision="$2"; shift 2;; + --regions) image_regions=("$2"); shift 2;; # parse as whitespace separated array + --size) box_size="$2"; shift 2;; + --box) box_name="$2"; destroy_droplet="no"; shift 2;; + --no-destroy) destroy_droplet="no"; shift 2;; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac +done + +# set DO token, picked up by digitalOceanFunctions.sh +if [[ "${deploy_env}" == "staging" ]]; then + assertNotEmpty DIGITAL_OCEAN_TOKEN_STAGING + readonly DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_STAGING}" +elif [[ "${deploy_env}" == "dev" ]]; then + assertNotEmpty DIGITAL_OCEAN_TOKEN_DEV + readonly DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_DEV}" +elif [[ "${deploy_env}" == "prod" ]]; then + assertNotEmpty DIGITAL_OCEAN_TOKEN_PROD + readonly DIGITAL_OCEAN_TOKEN="${DIGITAL_OCEAN_TOKEN_PROD}" +else + echo "No such env ${deploy_env}." + exit 1 +fi +source "${SCRIPT_DIR}/digitalOceanFunctions.sh" + +if [[ ! -f "${ssh_keys}" ]]; then + echo "yellowtent ssh key is missing" + exit 1 +fi + +function get_pretty_revision() { + local git_rev="$1" + local sha1=$(git rev-parse --short "${git_rev}" 2>/dev/null) + local name=$(git name-rev --name-only --tags "${sha1}" 2>/dev/null) + + if [[ -z "${name}" ]]; then + echo "Unable to resolve $1" + exit 1 + fi + + # fallback to sha1 if we cannot find a tag + if [[ "${name}" == "undefined" ]]; then + echo "${sha1}" + else + echo "${name}" + fi +} + +now=$(date "+%Y-%m-%d-%H%M%S") +pretty_revision=$(get_pretty_revision "${installer_revision}") + +if [[ -z "${box_name}" ]]; then + # if you change this, change the regexp is appstore/janitor.js + box_name="box-${deploy_env}-${pretty_revision}-${now}" # remove slashes + + # create a new droplet if no name given + yellowtent_ssh_key_id=$(get_ssh_key_id "yellowtent") + if [[ -z "${yellowtent_ssh_key_id}" ]]; then + echo "Could not query yellowtent ssh key" + exit 1 + fi + echo "Detected yellowtent ssh key id: ${yellowtent_ssh_key_id}" # 124654 for yellowtent key + + echo "Creating Droplet with name [${box_name}] at [${image_regions[0]}] with size [${box_size}]" + droplet_id=$(create_droplet ${yellowtent_ssh_key_id} ${box_name} ${box_size} ${image_regions[0]}) + if [[ -z "${droplet_id}" ]]; then + echo "Failed to create droplet" + exit 1 + fi + echo "Created droplet with id: ${droplet_id}" + + # If we run scripts overenthusiastically without the wait, setup script randomly fails + echo -n "Waiting 120 seconds for droplet creation" + for i in $(seq 1 24); do + echo -n "." + sleep 5 + done + echo "" +else + droplet_id=$(get_droplet_id "${box_name}") + echo "Reusing droplet with id: ${droplet_id}" + + power_on_droplet "${droplet_id}" +fi + +# Query DO until we get an IP +while true; do + echo "Trying to get the droplet IP" + droplet_ip=$(get_droplet_ip "${droplet_id}") + if [[ "${droplet_ip}" != "" ]]; then + echo "Droplet IP : [${droplet_ip}]" + break + fi + echo "Timedout, trying again in 10 seconds" + sleep 10 +done + +while true; do + echo "Trying to copy init script to droplet" + if scp -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${ssh_keys}" "${SCRIPT_DIR}/initializeBaseUbuntuImage.sh" root@${droplet_ip}:.; then + break + fi + echo "Timedout, trying again in 30 seconds" + sleep 30 +done + +echo "Copying installer source" +cd "${INSTALLER_DIR}" +git archive --format=tar HEAD | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${ssh_keys}" "root@${droplet_ip}" "cat - > /root/installer.tar" + +echo "Executing init script" +if ! ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${ssh_keys}" "root@${droplet_ip}" "/bin/bash /root/initializeBaseUbuntuImage.sh ${installer_revision}"; then + echo "Init script failed" + exit 1 +fi + +echo "Copy over certs" +scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${ssh_keys}" "${INSTALLER_DIR}/../keys/installer/" "root@${droplet_ip}:/home/yellowtent/installer/src/certs/" +scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${ssh_keys}" "${INSTALLER_DIR}/../keys/installer_ca/ca.crt" "root@${droplet_ip}:/home/yellowtent/installer/src/certs/" + +echo "Shutting down droplet with id : ${droplet_id}" +ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${ssh_keys}" "root@${droplet_ip}" "shutdown -f now" || true # shutdown sometimes terminates ssh connection immediately making this command fail + +# wait 10 secs for actual shutdown +echo "Waiting for 10 seconds for droplet to shutdown" +sleep 30 + +echo "Powering off droplet" +power_off_droplet "${droplet_id}" + +snapshot_name="box-${deploy_env}-${pretty_revision}-${now}" +echo "Snapshotting as ${snapshot_name}" +snapshot_droplet "${droplet_id}" "${snapshot_name}" + +image_id=$(get_image_id "${snapshot_name}") +echo "Image id is ${image_id}" + +if [[ "${destroy_droplet}" == "yes" ]]; then + echo "Destroying droplet" + destroy_droplet "${droplet_id}" +else + echo "Skipping droplet destroy" +fi + +echo "Transferring image to other regions" +xfer_events=() +# skip the first region, as the image was created there +for image_region in ${image_regions[@]:1}; do + xfer_event=$(transfer_image ${image_id} ${image_region}) + echo "Image transfer to ${image_region} initiated. Event id: ${xfer_event}" + xfer_events+=("${xfer_event}") + sleep 1 +done + +echo "Image transfer initiated, but they will take some time to get transferred." + +for xfer_event in ${xfer_events[@]}; do + wait_for_image_event "${image_id}" "${xfer_event}" +done + +echo "Done." + diff --git a/images/digitalOceanFunctions.sh b/images/digitalOceanFunctions.sh new file mode 100644 index 000000000..447973373 --- /dev/null +++ b/images/digitalOceanFunctions.sh @@ -0,0 +1,180 @@ +#!/bin/bash + +if [[ -z "${DIGITAL_OCEAN_TOKEN}" ]]; then + echo "Script requires DIGITAL_OCEAN_TOKEN env to be set" + exit 1 +fi + +if [[ -z "${JSON}" ]]; then + echo "Script requires JSON env to be set to path of JSON binary" + exit 1 +fi + +readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:" + +function get_ssh_key_id() { + $CURL "https://api.digitalocean.com/v2/account/keys" \ + | $JSON ssh_keys \ + | $JSON -c "this.name === \"$1\"" \ + | $JSON 0.id +} + +function create_droplet() { + local ssh_key_id="$1" + local box_name="$2" + local box_size="$3" + local image_region="$4" + + local ubuntu_image_slug="ubuntu-14-10-x64" + + local data="{\"name\":\"${box_name}\",\"size\":\"${box_size}\",\"region\":\"${image_region}\",\"image\":\"${ubuntu_image_slug}\",\"ssh_keys\":[ \"${ssh_key_id}\" ],\"backups\":false}" + + $CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets" | $JSON droplet.id +} + +function get_droplet_ip() { + local droplet_id="$1" + $CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}" | $JSON "droplet.networks.v4[0].ip_address" +} + +function get_droplet_id() { + local droplet_name="$1" + $CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id" +} + +function power_off_droplet() { + local droplet_id="$1" + local data='{"type":"power_off"}' + local response=$($CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions") + local event_id=`echo "${response}" | $JSON action.id` + + if [[ -z "${event_id}" ]]; then + echo "Got no event id, assuming already powered off." + echo "Response: ${response}" + return + fi + + echo "Powered off droplet. Event id: ${event_id}" + echo -n "Waiting for droplet to power off" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + echo -n "." + sleep 10 + done + echo "" +} + +function power_on_droplet() { + local droplet_id="$1" + local data='{"type":"power_on"}' + local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id` + + echo "Powered on droplet. Event id: ${event_id}" + + if [[ -z "${event_id}" ]]; then + echo "Got no event id, assuming already powered on" + return + fi + + echo -n "Waiting for droplet to power on" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + echo -n "." + sleep 10 + done + echo "" +} + +function snapshot_droplet() { + local droplet_id="$1" + local snapshot_name="$2" + local data="{\"type\":\"snapshot\",\"name\":\"${snapshot_name}\"}" + local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions" | $JSON action.id` + + echo "Droplet snapshotted as ${snapshot_name}. Event id: ${event_id}" + echo -n "Waiting for snapshot to complete" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + echo -n "." + sleep 10 + done + echo "" +} + +function destroy_droplet() { + local droplet_id="$1" + # TODO: check for 204 status + $CURL -X DELETE "https://api.digitalocean.com/v2/droplets/${droplet_id}" + echo "Droplet destroyed" + echo "" +} + +function transfer_image() { + local image_id="$1" + local region_slug="$2" + local data="{\"type\":\"transfer\",\"region\":\"${region_slug}\"}" + local event_id=`$CURL -X POST -H 'Content-Type: application/json' -d "${data}" "https://api.digitalocean.com/v2/images/${image_id}/actions" | $JSON action.id` + echo "${event_id}" +} + +function get_image_id() { + local snapshot_name="$1" + local image_id="" + + image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \ + | $JSON images \ + | $JSON -c "this.name === \"${snapshot_name}\"" 0.id) + + if [[ -n "${image_id}" ]]; then + echo "${image_id}" + fi +} + +function get_image_id_by_revision() { + local revision="$1" + local image_id="" + + image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \ + | $JSON images \ + | $JSON -c "this.name.indexOf(\"box-${revision}\") === 0" 0.id) + + if [[ -n "${image_id}" ]]; then + echo "${image_id}" + fi +} + +function get_image_name() { + local image_id="$1" + $CURL "https://api.digitalocean.com/v2/images/${image_id}?per_page=100" \ + | $JSON image.name +} + +function wait_for_image_event() { + local image_id="$1" + local event_id="$2" + + echo -n "Waiting for ${event_id}" + + while true; do + local event_status=`$CURL "https://api.digitalocean.com/v2/images/${image_id}/actions/${event_id}" | $JSON action.status` + if [[ "${event_status}" == "completed" ]]; then + break + fi + echo -n "." + sleep 10 + done + echo "" +} + diff --git a/images/initializeBaseUbuntuImage.sh b/images/initializeBaseUbuntuImage.sh new file mode 100755 index 000000000..87c2f881d --- /dev/null +++ b/images/initializeBaseUbuntuImage.sh @@ -0,0 +1,229 @@ +#!/bin/bash + +set -euv -o pipefail + +readonly USER=yellowtent +readonly USER_HOME="/home/${USER}" +readonly DATA_DIR="${USER_HOME}/data" +readonly APPDATA="${DATA_DIR}/appdata" +readonly INSTALLER_SOURCE_DIR="${USER_HOME}/installer" +readonly INSTALLER_REVISION="$1" +readonly DOCKER_DATA_FILE="/root/docker_data.img" +readonly USER_HOME_FILE="/root/user_home.img" + +echo "==== Create User ${USER} ====" +if ! id "${USER}"; then + useradd "${USER}" -m +fi + +echo "=== Yellowtent base image preparation (installer revision - ${INSTALLER_REVISION}) ===" + +export DEBIAN_FRONTEND=noninteractive + +# Allocate two sets of swap files - one for general app usage and another for backup +# The backup swap is setup for swap on the fly by the backup scripts +echo "=== Setup swap file ===" +apps_swap_file="/apps.swap" +[[ -f "${apps_swap_file}" ]] && swapoff "${apps_swap_file}" +fallocate -l 1024m "${apps_swap_file}" +chmod 600 "${apps_swap_file}" +mkswap "${apps_swap_file}" +swapon "${apps_swap_file}" +echo "${apps_swap_file} none swap sw 0 0" >> /etc/fstab + +backup_swap_file="/backup.swap" +[[ -f "${backup_swap_file}" ]] && swapoff "${backup_swap_file}" +fallocate -l 1024m "${backup_swap_file}" +chmod 600 "${backup_swap_file}" +mkswap "${backup_swap_file}" + +echo "==== Install project dependencies ====" +apt-get update + +echo "=== Upgrade ===" +apt-get upgrade -y + +# Setup firewall before everything. Atleast docker 1.5 creates it's own chain and the -X below will remove it +# Do NOT use iptables-persistent because it's startup ordering conflicts with docker +echo "=== Setting up firewall ===" +# clear tables and set default policy +iptables -F # flush all chains +iptables -X # delete all chains +# default policy for filter table +iptables -P INPUT DROP +iptables -P FORWARD ACCEPT # TODO: disable icc and make this as reject +iptables -P OUTPUT ACCEPT + +# NOTE: keep these in sync with src/apps.js validatePortBindings +# allow ssh, http, https, ping, dns +iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT +iptables -A INPUT -p tcp --dport 22 -j ACCEPT +iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,443,886 -j ACCEPT +iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT +iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT +iptables -A INPUT -p udp --sport 53 -j ACCEPT +iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP: + +# loopback +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT + +# disable metadata access to non-root +# modprobe ipt_owner +iptables -A OUTPUT -m owner ! --uid-owner root -d 169.254.169.254 -j DROP + +# prevent DoS +# iptables -A INPUT -p tcp --dport 80 -m limit --limit 25/minute --limit-burst 100 -j ACCEPT + +# log dropped incoming. keep this at the end of all the rules +iptables -N LOGGING # new chain +iptables -A INPUT -j LOGGING # last rule in INPUT chain +iptables -A LOGGING -m limit --limit 2/min -j LOG --log-prefix "IPTables Packet Dropped: " --log-level 7 +iptables -A LOGGING -j DROP + +echo "==== Install docker ====" +# see http://idolstarastronomer.com/painless-docker.html +echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 +apt-get update +apt-get -y install lxc-docker-1.5.0 +ln -sf /usr/bin/docker.io /usr/local/bin/docker + +if [ ! -f "${DOCKER_DATA_FILE}" ]; then + service docker stop + if aufs_mounts=$(grep 'aufs' /proc/mounts | awk '{print$2}' | sort -r); then + umount -l "${aufs_mounts}" + fi + rm -rf /var/lib/docker + mkdir /var/lib/docker + + # create a separate 12GB fs for docker images + # dd if=/dev/zero of=/root/docker_data.img bs=1M count=12000 + apt-get -y install btrfs-tools + truncate -s 12G "${DOCKER_DATA_FILE}" + mkfs.btrfs -L DockerData "${DOCKER_DATA_FILE}" + echo "${DOCKER_DATA_FILE} /var/lib/docker btrfs loop,nosuid 0 0" >> /etc/fstab + echo 'DOCKER_OPTS="-s btrfs"' >> /etc/default/docker + mount "${DOCKER_DATA_FILE}" + + service docker start + # give docker sometime to start up and create iptables rules + sleep 10 +fi + +# ubuntu will restore iptables from this file automatically. this is here so that docker's chain is saved to this file +mkdir /etc/iptables && iptables-save > /etc/iptables/rules.v4 + +# now add the user to the docker group +usermod "${USER}" -a -G docker +echo "=== Pulling base docker images ===" +docker pull cloudron/base:0.3.0 + +echo "=== Pulling mysql addon image ===" +docker pull cloudron/mysql:0.3.0 + +echo "=== Pulling postgresql addon image ===" +docker pull cloudron/postgresql:0.3.0 + +echo "=== Pulling redis addon image ===" +docker pull cloudron/redis:0.3.0 + +echo "=== Pulling mongodb addon image ===" +docker pull cloudron/mongodb:0.3.0 + +echo "=== Pulling graphite docker images ===" +docker pull cloudron/graphite:0.3.1 + +echo "=== Pulling mail relay ===" +docker pull cloudron/mail:0.3.0 + +echo "==== Install nginx ====" +apt-get -y install nginx-full + +echo "==== Install build-essential ====" +apt-get -y install build-essential rcconf + + +echo "==== Install mysql ====" +debconf-set-selections <<< 'mysql-server mysql-server/root_password password password' +debconf-set-selections <<< 'mysql-server mysql-server/root_password_again password password' +apt-get -y install mysql-server + +echo "==== Install pwgen ====" +apt-get -y install pwgen + +echo "==== Install supervisor ====" +apt-get -y install supervisor + +echo "==== Install collectd ===" +apt-get install -y collectd collectd-utils +update-rc.d -f collectd remove + +echo "==== Seting up btrfs user home ===" +if [[ ! -f "${USER_HOME_FILE}" ]]; then + # create a separate 12GB fs for data + truncate -s 12G "${USER_HOME_FILE}" + mkfs.btrfs -L UserHome "${USER_HOME_FILE}" + echo "${USER_HOME_FILE} ${USER_HOME} btrfs loop,nosuid 0 0" >> /etc/fstab + mount "${USER_HOME_FILE}" + btrfs subvolume create "${USER_HOME}/data" +fi + +echo "=== Install tmpreaper ===" +sudo apt-get install -y tmpreaper +sudo sed -e 's/SHOWWARNING=true/# SHOWWARNING=true/' -i /etc/tmpreaper.conf + +echo "==== Extracting installer source ====" +rm -rf "${INSTALLER_SOURCE_DIR}" && mkdir -p "${INSTALLER_SOURCE_DIR}" +tar xvf /root/installer.tar -C "${INSTALLER_SOURCE_DIR}" && rm /root/installer.tar +echo "${INSTALLER_REVISION}" > "${INSTALLER_SOURCE_DIR}/REVISION" + +echo "==== Install nodejs ====" +apt-get install -y curl +curl -sL https://deb.nodesource.com/setup_0.12 | bash - +apt-get install -y nodejs + +echo "=== Rebuilding npm packages ===" +cd "${INSTALLER_SOURCE_DIR}" && npm install --production + +echo "==== Make the user own his home ====" +chown "${USER}:${USER}" -R "/home/${USER}" + +echo "==== Install init script ====" +cat > /etc/init.d/cloudron-bootstrap <&2 + exit 3 + ;; +esac +EOF + +chmod +x /etc/init.d/cloudron-bootstrap +update-rc.d cloudron-bootstrap defaults 99 + +sync + diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json new file mode 100644 index 000000000..aed99cbf2 --- /dev/null +++ b/npm-shrinkwrap.json @@ -0,0 +1,923 @@ +{ + "name": "installer", + "version": "0.0.1", + "dependencies": { + "async": { + "version": "0.9.0", + "from": "https://registry.npmjs.org/async/-/async-0.9.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.0.tgz" + }, + "body-parser": { + "version": "1.12.2", + "from": "https://registry.npmjs.org/body-parser/-/body-parser-1.12.2.tgz", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.12.2.tgz", + "dependencies": { + "bytes": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/bytes/-/bytes-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/bytes/-/bytes-1.0.0.tgz" + }, + "content-type": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz" + }, + "depd": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/depd/-/depd-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.0.tgz" + }, + "iconv-lite": { + "version": "0.4.7", + "from": "http://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.7.tgz", + "resolved": "http://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.7.tgz" + }, + "on-finished": { + "version": "2.2.0", + "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.2.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.0", + "from": "http://registry.npmjs.org/ee-first/-/ee-first-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/ee-first/-/ee-first-1.1.0.tgz" + } + } + }, + "qs": { + "version": "2.4.1", + "from": "https://registry.npmjs.org/qs/-/qs-2.4.1.tgz", + "resolved": "https://registry.npmjs.org/qs/-/qs-2.4.1.tgz" + }, + "raw-body": { + "version": "1.3.3", + "from": "http://registry.npmjs.org/raw-body/-/raw-body-1.3.3.tgz", + "resolved": "http://registry.npmjs.org/raw-body/-/raw-body-1.3.3.tgz" + }, + "type-is": { + "version": "1.6.1", + "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.1.tgz", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.1.tgz", + "dependencies": { + "media-typer": { + "version": "0.3.0", + "from": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "mime-types": { + "version": "2.0.10", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "dependencies": { + "mime-db": { + "version": "1.8.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz" + } + } + } + } + } + } + }, + "connect-lastmile": { + "version": "0.0.10", + "from": "http://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.10.tgz", + "resolved": "http://registry.npmjs.org/connect-lastmile/-/connect-lastmile-0.0.10.tgz" + }, + "debug": { + "version": "2.1.3", + "from": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.1.3.tgz", + "dependencies": { + "ms": { + "version": "0.7.0", + "from": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz", + "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz" + } + } + }, + "express": { + "version": "4.12.3", + "from": "https://registry.npmjs.org/express/-/express-4.12.3.tgz", + "resolved": "https://registry.npmjs.org/express/-/express-4.12.3.tgz", + "dependencies": { + "accepts": { + "version": "1.2.5", + "from": "https://registry.npmjs.org/accepts/-/accepts-1.2.5.tgz", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.5.tgz", + "dependencies": { + "mime-types": { + "version": "2.0.10", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "dependencies": { + "mime-db": { + "version": "1.8.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz" + } + } + }, + "negotiator": { + "version": "0.5.1", + "from": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.1.tgz" + } + } + }, + "content-disposition": { + "version": "0.5.0", + "from": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz", + "resolved": "http://registry.npmjs.org/content-disposition/-/content-disposition-0.5.0.tgz" + }, + "content-type": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.1.tgz" + }, + "cookie": { + "version": "0.1.2", + "from": "https://registry.npmjs.org/cookie/-/cookie-0.1.2.tgz", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.2.tgz" + }, + "cookie-signature": { + "version": "1.0.6", + "from": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + }, + "depd": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/depd/-/depd-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.0.tgz" + }, + "escape-html": { + "version": "1.0.1", + "from": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/escape-html/-/escape-html-1.0.1.tgz" + }, + "etag": { + "version": "1.5.1", + "from": "http://registry.npmjs.org/etag/-/etag-1.5.1.tgz", + "resolved": "http://registry.npmjs.org/etag/-/etag-1.5.1.tgz", + "dependencies": { + "crc": { + "version": "3.2.1", + "from": "http://registry.npmjs.org/crc/-/crc-3.2.1.tgz", + "resolved": "http://registry.npmjs.org/crc/-/crc-3.2.1.tgz" + } + } + }, + "finalhandler": { + "version": "0.3.4", + "from": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.3.4.tgz", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-0.3.4.tgz" + }, + "fresh": { + "version": "0.2.4", + "from": "http://registry.npmjs.org/fresh/-/fresh-0.2.4.tgz", + "resolved": "http://registry.npmjs.org/fresh/-/fresh-0.2.4.tgz" + }, + "merge-descriptors": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz" + }, + "methods": { + "version": "1.1.1", + "from": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz" + }, + "on-finished": { + "version": "2.2.0", + "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.2.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.0", + "from": "http://registry.npmjs.org/ee-first/-/ee-first-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/ee-first/-/ee-first-1.1.0.tgz" + } + } + }, + "parseurl": { + "version": "1.3.0", + "from": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz", + "resolved": "http://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz" + }, + "path-to-regexp": { + "version": "0.1.3", + "from": "http://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.3.tgz", + "resolved": "http://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.3.tgz" + }, + "proxy-addr": { + "version": "1.0.7", + "from": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.7.tgz", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.7.tgz", + "dependencies": { + "forwarded": { + "version": "0.1.0", + "from": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz", + "resolved": "http://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz" + }, + "ipaddr.js": { + "version": "0.1.9", + "from": "http://registry.npmjs.org/ipaddr.js/-/ipaddr.js-0.1.9.tgz", + "resolved": "http://registry.npmjs.org/ipaddr.js/-/ipaddr.js-0.1.9.tgz" + } + } + }, + "qs": { + "version": "2.4.1", + "from": "https://registry.npmjs.org/qs/-/qs-2.4.1.tgz", + "resolved": "https://registry.npmjs.org/qs/-/qs-2.4.1.tgz" + }, + "range-parser": { + "version": "1.0.2", + "from": "http://registry.npmjs.org/range-parser/-/range-parser-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/range-parser/-/range-parser-1.0.2.tgz" + }, + "send": { + "version": "0.12.2", + "from": "https://registry.npmjs.org/send/-/send-0.12.2.tgz", + "resolved": "https://registry.npmjs.org/send/-/send-0.12.2.tgz", + "dependencies": { + "destroy": { + "version": "1.0.3", + "from": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/destroy/-/destroy-1.0.3.tgz" + }, + "mime": { + "version": "1.3.4", + "from": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz" + }, + "ms": { + "version": "0.7.0", + "from": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz", + "resolved": "http://registry.npmjs.org/ms/-/ms-0.7.0.tgz" + } + } + }, + "serve-static": { + "version": "1.9.2", + "from": "https://registry.npmjs.org/serve-static/-/serve-static-1.9.2.tgz", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.9.2.tgz" + }, + "type-is": { + "version": "1.6.1", + "from": "https://registry.npmjs.org/type-is/-/type-is-1.6.1.tgz", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.1.tgz", + "dependencies": { + "media-typer": { + "version": "0.3.0", + "from": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + }, + "mime-types": { + "version": "2.0.10", + "from": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.10.tgz", + "dependencies": { + "mime-db": { + "version": "1.8.0", + "from": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.8.0.tgz" + } + } + } + } + }, + "vary": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/vary/-/vary-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/vary/-/vary-1.0.0.tgz" + }, + "utils-merge": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz" + } + } + }, + "forever": { + "version": "0.14.1", + "from": "http://registry.npmjs.org/forever/-/forever-0.14.1.tgz", + "resolved": "http://registry.npmjs.org/forever/-/forever-0.14.1.tgz", + "dependencies": { + "colors": { + "version": "0.6.2", + "from": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz" + }, + "cliff": { + "version": "0.1.10", + "from": "http://registry.npmjs.org/cliff/-/cliff-0.1.10.tgz", + "resolved": "http://registry.npmjs.org/cliff/-/cliff-0.1.10.tgz", + "dependencies": { + "colors": { + "version": "1.0.3", + "from": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz" + }, + "eyes": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" + } + } + }, + "flatiron": { + "version": "0.4.3", + "from": "http://registry.npmjs.org/flatiron/-/flatiron-0.4.3.tgz", + "resolved": "http://registry.npmjs.org/flatiron/-/flatiron-0.4.3.tgz", + "dependencies": { + "broadway": { + "version": "0.3.6", + "from": "http://registry.npmjs.org/broadway/-/broadway-0.3.6.tgz", + "resolved": "http://registry.npmjs.org/broadway/-/broadway-0.3.6.tgz", + "dependencies": { + "cliff": { + "version": "0.1.9", + "from": "http://registry.npmjs.org/cliff/-/cliff-0.1.9.tgz", + "resolved": "http://registry.npmjs.org/cliff/-/cliff-0.1.9.tgz", + "dependencies": { + "eyes": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" + } + } + }, + "eventemitter2": { + "version": "0.4.14", + "from": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz" + }, + "winston": { + "version": "0.8.0", + "from": "http://registry.npmjs.org/winston/-/winston-0.8.0.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-0.8.0.tgz", + "dependencies": { + "async": { + "version": "0.2.10", + "from": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" + }, + "cycle": { + "version": "1.0.3", + "from": "http://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz" + }, + "eyes": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" + }, + "pkginfo": { + "version": "0.3.0", + "from": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz" + }, + "stack-trace": { + "version": "0.0.9", + "from": "http://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", + "resolved": "http://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" + } + } + } + } + }, + "optimist": { + "version": "0.6.0", + "from": "http://registry.npmjs.org/optimist/-/optimist-0.6.0.tgz", + "resolved": "http://registry.npmjs.org/optimist/-/optimist-0.6.0.tgz", + "dependencies": { + "wordwrap": { + "version": "0.0.2", + "from": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", + "resolved": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz" + }, + "minimist": { + "version": "0.0.10", + "from": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz" + } + } + }, + "prompt": { + "version": "0.2.14", + "from": "http://registry.npmjs.org/prompt/-/prompt-0.2.14.tgz", + "resolved": "http://registry.npmjs.org/prompt/-/prompt-0.2.14.tgz", + "dependencies": { + "pkginfo": { + "version": "0.3.0", + "from": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz" + }, + "read": { + "version": "1.0.5", + "from": "http://registry.npmjs.org/read/-/read-1.0.5.tgz", + "resolved": "http://registry.npmjs.org/read/-/read-1.0.5.tgz", + "dependencies": { + "mute-stream": { + "version": "0.0.4", + "from": "http://registry.npmjs.org/mute-stream/-/mute-stream-0.0.4.tgz", + "resolved": "http://registry.npmjs.org/mute-stream/-/mute-stream-0.0.4.tgz" + } + } + }, + "revalidator": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/revalidator/-/revalidator-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/revalidator/-/revalidator-0.1.8.tgz" + } + } + }, + "director": { + "version": "1.2.7", + "from": "http://registry.npmjs.org/director/-/director-1.2.7.tgz", + "resolved": "http://registry.npmjs.org/director/-/director-1.2.7.tgz" + } + } + }, + "forever-monitor": { + "version": "1.5.2", + "from": "http://registry.npmjs.org/forever-monitor/-/forever-monitor-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/forever-monitor/-/forever-monitor-1.5.2.tgz", + "dependencies": { + "broadway": { + "version": "0.3.6", + "from": "http://registry.npmjs.org/broadway/-/broadway-0.3.6.tgz", + "resolved": "http://registry.npmjs.org/broadway/-/broadway-0.3.6.tgz", + "dependencies": { + "cliff": { + "version": "0.1.9", + "from": "http://registry.npmjs.org/cliff/-/cliff-0.1.9.tgz", + "resolved": "http://registry.npmjs.org/cliff/-/cliff-0.1.9.tgz", + "dependencies": { + "eyes": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" + } + } + }, + "eventemitter2": { + "version": "0.4.14", + "from": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz" + }, + "winston": { + "version": "0.8.0", + "from": "http://registry.npmjs.org/winston/-/winston-0.8.0.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-0.8.0.tgz", + "dependencies": { + "async": { + "version": "0.2.10", + "from": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" + }, + "cycle": { + "version": "1.0.3", + "from": "http://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz" + }, + "eyes": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" + }, + "pkginfo": { + "version": "0.3.0", + "from": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz" + }, + "stack-trace": { + "version": "0.0.9", + "from": "http://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", + "resolved": "http://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" + } + } + } + } + }, + "minimatch": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/minimatch/-/minimatch-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/minimatch/-/minimatch-1.0.0.tgz", + "dependencies": { + "lru-cache": { + "version": "2.5.0", + "from": "http://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz", + "resolved": "http://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz" + }, + "sigmund": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/sigmund/-/sigmund-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/sigmund/-/sigmund-1.0.0.tgz" + } + } + }, + "ps-tree": { + "version": "0.0.3", + "from": "http://registry.npmjs.org/ps-tree/-/ps-tree-0.0.3.tgz", + "resolved": "http://registry.npmjs.org/ps-tree/-/ps-tree-0.0.3.tgz", + "dependencies": { + "event-stream": { + "version": "0.5.3", + "from": "http://registry.npmjs.org/event-stream/-/event-stream-0.5.3.tgz", + "resolved": "http://registry.npmjs.org/event-stream/-/event-stream-0.5.3.tgz", + "dependencies": { + "optimist": { + "version": "0.2.8", + "from": "http://registry.npmjs.org/optimist/-/optimist-0.2.8.tgz", + "resolved": "http://registry.npmjs.org/optimist/-/optimist-0.2.8.tgz", + "dependencies": { + "wordwrap": { + "version": "0.0.2", + "from": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", + "resolved": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz" + } + } + } + } + } + } + }, + "watch": { + "version": "0.13.0", + "from": "http://registry.npmjs.org/watch/-/watch-0.13.0.tgz", + "resolved": "http://registry.npmjs.org/watch/-/watch-0.13.0.tgz", + "dependencies": { + "minimist": { + "version": "1.1.1", + "from": "http://registry.npmjs.org/minimist/-/minimist-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.1.1.tgz" + } + } + } + } + }, + "nconf": { + "version": "0.6.9", + "from": "http://registry.npmjs.org/nconf/-/nconf-0.6.9.tgz", + "resolved": "http://registry.npmjs.org/nconf/-/nconf-0.6.9.tgz", + "dependencies": { + "async": { + "version": "0.2.9", + "from": "http://registry.npmjs.org/async/-/async-0.2.9.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-0.2.9.tgz" + }, + "ini": { + "version": "1.3.3", + "from": "http://registry.npmjs.org/ini/-/ini-1.3.3.tgz", + "resolved": "http://registry.npmjs.org/ini/-/ini-1.3.3.tgz" + }, + "optimist": { + "version": "0.6.0", + "from": "http://registry.npmjs.org/optimist/-/optimist-0.6.0.tgz", + "resolved": "http://registry.npmjs.org/optimist/-/optimist-0.6.0.tgz", + "dependencies": { + "wordwrap": { + "version": "0.0.2", + "from": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", + "resolved": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz" + }, + "minimist": { + "version": "0.0.10", + "from": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz" + } + } + } + } + }, + "nssocket": { + "version": "0.5.3", + "from": "http://registry.npmjs.org/nssocket/-/nssocket-0.5.3.tgz", + "resolved": "http://registry.npmjs.org/nssocket/-/nssocket-0.5.3.tgz", + "dependencies": { + "eventemitter2": { + "version": "0.4.14", + "from": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz" + }, + "lazy": { + "version": "1.0.11", + "from": "http://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz", + "resolved": "http://registry.npmjs.org/lazy/-/lazy-1.0.11.tgz" + } + } + }, + "optimist": { + "version": "0.6.1", + "from": "http://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "resolved": "http://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", + "dependencies": { + "wordwrap": { + "version": "0.0.2", + "from": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz", + "resolved": "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz" + }, + "minimist": { + "version": "0.0.10", + "from": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz" + } + } + }, + "timespan": { + "version": "2.3.0", + "from": "http://registry.npmjs.org/timespan/-/timespan-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/timespan/-/timespan-2.3.0.tgz" + }, + "utile": { + "version": "0.2.1", + "from": "http://registry.npmjs.org/utile/-/utile-0.2.1.tgz", + "resolved": "http://registry.npmjs.org/utile/-/utile-0.2.1.tgz", + "dependencies": { + "async": { + "version": "0.2.10", + "from": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" + }, + "deep-equal": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/deep-equal/-/deep-equal-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/deep-equal/-/deep-equal-1.0.0.tgz" + }, + "i": { + "version": "0.3.3", + "from": "http://registry.npmjs.org/i/-/i-0.3.3.tgz", + "resolved": "http://registry.npmjs.org/i/-/i-0.3.3.tgz" + }, + "mkdirp": { + "version": "0.5.0", + "from": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz", + "dependencies": { + "minimist": { + "version": "0.0.8", + "from": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + } + } + }, + "ncp": { + "version": "0.4.2", + "from": "http://registry.npmjs.org/ncp/-/ncp-0.4.2.tgz", + "resolved": "http://registry.npmjs.org/ncp/-/ncp-0.4.2.tgz" + }, + "rimraf": { + "version": "2.3.2", + "from": "https://registry.npmjs.org/rimraf/-/rimraf-2.3.2.tgz", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.3.2.tgz", + "dependencies": { + "glob": { + "version": "4.5.3", + "from": "http://registry.npmjs.org/glob/-/glob-4.5.3.tgz", + "resolved": "http://registry.npmjs.org/glob/-/glob-4.5.3.tgz", + "dependencies": { + "inflight": { + "version": "1.0.4", + "from": "http://registry.npmjs.org/inflight/-/inflight-1.0.4.tgz", + "resolved": "http://registry.npmjs.org/inflight/-/inflight-1.0.4.tgz", + "dependencies": { + "wrappy": { + "version": "1.0.1", + "from": "http://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" + } + } + }, + "inherits": { + "version": "2.0.1", + "from": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, + "minimatch": { + "version": "2.0.4", + "from": "http://registry.npmjs.org/minimatch/-/minimatch-2.0.4.tgz", + "resolved": "http://registry.npmjs.org/minimatch/-/minimatch-2.0.4.tgz", + "dependencies": { + "brace-expansion": { + "version": "1.1.0", + "from": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.0.tgz", + "dependencies": { + "balanced-match": { + "version": "0.2.0", + "from": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.2.0.tgz", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.2.0.tgz" + }, + "concat-map": { + "version": "0.0.1", + "from": "http://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "resolved": "http://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + } + } + } + } + }, + "once": { + "version": "1.3.1", + "from": "http://registry.npmjs.org/once/-/once-1.3.1.tgz", + "resolved": "http://registry.npmjs.org/once/-/once-1.3.1.tgz", + "dependencies": { + "wrappy": { + "version": "1.0.1", + "from": "http://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/wrappy/-/wrappy-1.0.1.tgz" + } + } + } + } + } + } + } + } + }, + "winston": { + "version": "0.8.3", + "from": "http://registry.npmjs.org/winston/-/winston-0.8.3.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-0.8.3.tgz", + "dependencies": { + "async": { + "version": "0.2.10", + "from": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" + }, + "cycle": { + "version": "1.0.3", + "from": "http://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz" + }, + "eyes": { + "version": "0.1.8", + "from": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "resolved": "http://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz" + }, + "isstream": { + "version": "0.1.2", + "from": "http://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "resolved": "http://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + }, + "pkginfo": { + "version": "0.3.0", + "from": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/pkginfo/-/pkginfo-0.3.0.tgz" + }, + "stack-trace": { + "version": "0.0.9", + "from": "http://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", + "resolved": "http://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" + } + } + } + } + }, + "json": { + "version": "9.0.3", + "from": "https://registry.npmjs.org/json/-/json-9.0.3.tgz", + "resolved": "https://registry.npmjs.org/json/-/json-9.0.3.tgz" + }, + "morgan": { + "version": "1.5.2", + "from": "https://registry.npmjs.org/morgan/-/morgan-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.5.2.tgz", + "dependencies": { + "basic-auth": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/basic-auth/-/basic-auth-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/basic-auth/-/basic-auth-1.0.0.tgz" + }, + "depd": { + "version": "1.0.0", + "from": "http://registry.npmjs.org/depd/-/depd-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/depd/-/depd-1.0.0.tgz" + }, + "on-finished": { + "version": "2.2.0", + "from": "https://registry.npmjs.org/on-finished/-/on-finished-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.2.0.tgz", + "dependencies": { + "ee-first": { + "version": "1.1.0", + "from": "http://registry.npmjs.org/ee-first/-/ee-first-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/ee-first/-/ee-first-1.1.0.tgz" + } + } + } + } + }, + "proxy-middleware": { + "version": "0.11.0", + "from": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.11.0.tgz", + "resolved": "https://registry.npmjs.org/proxy-middleware/-/proxy-middleware-0.11.0.tgz" + }, + "safetydance": { + "version": "0.0.16", + "from": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.16.tgz", + "resolved": "http://registry.npmjs.org/safetydance/-/safetydance-0.0.16.tgz" + }, + "superagent": { + "version": "0.21.0", + "from": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-0.21.0.tgz", + "dependencies": { + "qs": { + "version": "1.2.0", + "from": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/qs/-/qs-1.2.0.tgz" + }, + "formidable": { + "version": "1.0.14", + "from": "http://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz", + "resolved": "http://registry.npmjs.org/formidable/-/formidable-1.0.14.tgz" + }, + "mime": { + "version": "1.2.11", + "from": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + }, + "component-emitter": { + "version": "1.1.2", + "from": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz", + "resolved": "http://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz" + }, + "methods": { + "version": "1.0.1", + "from": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.0.1.tgz" + }, + "cookiejar": { + "version": "2.0.1", + "from": "http://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/cookiejar/-/cookiejar-2.0.1.tgz" + }, + "reduce-component": { + "version": "1.0.1", + "from": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/reduce-component/-/reduce-component-1.0.1.tgz" + }, + "extend": { + "version": "1.2.1", + "from": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz", + "resolved": "https://registry.npmjs.org/extend/-/extend-1.2.1.tgz" + }, + "form-data": { + "version": "0.1.3", + "from": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz", + "resolved": "http://registry.npmjs.org/form-data/-/form-data-0.1.3.tgz", + "dependencies": { + "combined-stream": { + "version": "0.0.7", + "from": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", + "dependencies": { + "delayed-stream": { + "version": "0.0.5", + "from": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", + "resolved": "http://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz" + } + } + } + } + }, + "readable-stream": { + "version": "1.0.27-1", + "from": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.27-1.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.1", + "from": "http://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/core-util-is/-/core-util-is-1.0.1.tgz" + }, + "isarray": { + "version": "0.0.1", + "from": "http://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "resolved": "http://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "inherits": { + "version": "2.0.1", + "from": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", + "resolved": "http://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + } + } + } + } + }, + "tail-stream": { + "version": "0.2.1", + "from": "http://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz", + "resolved": "http://registry.npmjs.org/tail-stream/-/tail-stream-0.2.1.tgz" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 000000000..14c4d5bd1 --- /dev/null +++ b/package.json @@ -0,0 +1,52 @@ +{ + "name": "installer", + "description": "Cloudron Installer", + "version": "0.0.1", + "private": "true", + "author": { + "name": "Cloudron authors" + }, + "repository": { + "type": "git" + }, + "engines": [ + "node >= 0.10.0" + ], + "dependencies": { + "async": "^0.9.0", + "body-parser": "^1.12.0", + "connect-lastmile": "0.0.10", + "debug": "^2.1.1", + "express": "^4.11.2", + "forever": "^0.14.1", + "json": "^9.0.3", + "morgan": "^1.5.1", + "proxy-middleware": "^0.11.0", + "safetydance": "0.0.16", + "superagent": "^0.21.0", + "tail-stream": "^0.2.1" + }, + "devDependencies": { + "aws-sdk": "^2.1.10", + "colors": "^1.0.3", + "commander": "^2.6.0", + "easy-table": "^0.3.0", + "expect.js": "^0.3.1", + "istanbul": "^0.3.5", + "lodash": "^3.2.0", + "mocha": "^2.1.0", + "nock": "^0.59.1", + "postmark": "^1.0.0", + "readline-sync": "^0.8.0", + "semver": "^4.3.0", + "ssh2": "^0.4.6", + "supererror": "^0.6.0", + "yesno": "0.0.1" + }, + "scripts": { + "test": "NODE_ENV=test ./node_modules/istanbul/lib/cli.js test $1 ./node_modules/mocha/bin/_mocha -- -R spec ./src/test", + "precommit": "/bin/true", + "prepush": "npm test", + "postmerge": "/bin/true" + } +} diff --git a/release/CHANGES b/release/CHANGES new file mode 100644 index 000000000..f90ad0b9d --- /dev/null +++ b/release/CHANGES @@ -0,0 +1,85 @@ +[0.0.1] +- Hot Chocolate + +[0.0.2] +- Hotfix appstore ui in webadim + +[0.0.3] +- Tall Pike + +[0.0.4] +- This will be 0.0.4 changes + +[0.0.5] +- App install/configure route fixes + +[0.0.6] +- Not sure what happenned here + +[0.0.7] +- resetToken is now sent as part of create user +- Same as 0.0.7 which got released by mistake + +[0.0.8] +- Manifest changes + +[0.0.9] +- Fix app restore +- Fix backup issues + +[0.0.10] +- Unknown orchestra + +[0.0.11] +- Add ldap addon + +[0.0.12] +- Support OAuth2 state + +[0.0.13] +- Use docker image from cloudron repository + +[0.0.14] +- Improve setup flow + +[0.0.15] +- Improved Appstore view + +[0.0.16] +- Improved Backup approach + +[0.0.17] +- Upgrade testing +- App auto updates +- Usage graphs + +[0.0.18] +- Rework backups and updates + +[0.0.19] +- Graphite fixes +- Avatar and Cloudron name support + +[0.0.20] +- Apptask fixes +- Chrome related fixes + +[0.0.21] +- Increase nginx hostname size to 64 + +[0.0.22] +- Testing the e2e tests + +[0.0.23] +- Better error status page +- Fix updater and backup progress reporting +- New avatar set +- Improved setup wizard + +[0.0.24] +- Hotfix the ldap support + +[0.0.25] +- Add support page +- Really fix ldap issues + diff --git a/release/images b/release/images new file mode 100755 index 000000000..b03c9c344 --- /dev/null +++ b/release/images @@ -0,0 +1,216 @@ +#!/usr/bin/env node + +'use strict'; + +require('supererror')({ splatchError: true }); + +var superagent = require('superagent'), + async = require('async'), + yesno = require('yesno'), + p = require('commander'); + +var DIGITALOCEAN = 'https://api.digitalocean.com/v2'; + +p.version('0.0.1') + .option('-l, --list', 'List images (default if neither --list or --cleanup provided') + .option('--cleanup', 'Delete images, which are not part of an release') + .option('-a, --all', 'Images from all environments (default if no argument provided)') + .option('-d, --development', 'Images from development') + .option('-s, --staging', 'Images from staging') + .option('-p, --production', 'Images from production') + .parse(process.argv); + +if (p.list || !p.cleanup) { + p.list = true; +} + +if (p.all || !(p.development || p.staging || p.production)) { + p.development = true; + p.staging = true; + p.production = true; +} + +function deleteImage(image, token, callback) { + var url = DIGITALOCEAN + '/images/' + image.id; + + console.log('Deleting image %s ...', image.name); + + superagent.del(url).set('Authorization', 'Bearer ' + token).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + callback(null); + }); +} + +function listImages(token, callback) { + var images = []; + var nextPage = null; + + async.doWhilst(function (callback) { + var url = DIGITALOCEAN + '/images?private=true'; + + superagent.get(url).set('Authorization', 'Bearer ' + token).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + nextPage = (result.body.links && result.body.links.pages && nextPage !== result.body.links.pages.next) ? result.body.links.pages.next : null; + images = images.concat(result.body.images); + + callback(null); + }); + }, function () { return !!nextPage; }, function (error) { + if (error) return callback(error); + callback(null, images); + }); +} + +function printEnvironment(tag, items, releases, callback) { + console.log(''); + console.log('%s:', tag); + console.log(''); + + var imageRegExp = new RegExp('box-(?:dev|staging|prod)-[0-9,a-f]{7}-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}'); + + items.forEach(function (item) { + if (!imageRegExp.test(item.name)) return; + + var releaseNumber = []; + for (var release in releases) { + if (releases.hasOwnProperty(release)) { + if (releases[release].imageId === item.id) { + releaseNumber.push(release); + } + } + } + + console.log(' %s : %s %s\t[%s]', item.id, item.name, releaseNumber.length ? releaseNumber.join(', ') : ' ', item.regions); + }); + + console.log(''); + + callback(null); +} + +function cleanupEnvironment(env, tag, items, releases, callback) { + console.log(''); + console.log('Cleanup images on %s:', tag); + + var imagesToCleanup = []; + + var imageRegExp = new RegExp('box-(?:dev|staging|prod)-[0-9,a-f]{7}-[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}'); + + items.forEach(function (item) { + if (!imageRegExp.test(item.name)) return; + + for (var release in releases) { + if (releases.hasOwnProperty(release)) { + if (releases[release].imageId === item.id) { + return; + } + } + } + + // we reached here so no release found + imagesToCleanup.push(item); + }); + + if (imagesToCleanup.length === 0) { + console.log('All images belong to a release.'); + return callback(null); + } + + imagesToCleanup.forEach(function (item) { + console.log(' %s : %s [%s]', item.id, item.name, item.regions); + }); + + console.log(''); + + yesno.ask('Do you want to delete those images? [y/N]', false, function (ok) { + if (ok) { + async.each(imagesToCleanup, function (image, callback) { + deleteImage(image, process.env[env], callback); + }, callback); + return; + } + + callback(null); + }); +} + + +function handleListEnvironment(active, env, tag, releaseUrl) { + return function (callback) { + if (!active) return callback(null); + if (!process.env[env]) { + console.log('%s not set. Skipping %s.', env, tag); + return callback(null); + } + + listImages(process.env[env], function (error, result) { + if (error) return callback(error); + + var images = result; + superagent.get(releaseUrl).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + // we get it as text + var releases = JSON.parse(result.text); + + printEnvironment(tag, images, releases, callback); + }); + }); + }; +} + +function handleCleanupEnvironment(active, env, tag, releaseUrl) { + return function (callback) { + if (!active) return callback(null); + if (!process.env[env]) { + console.log('%s not set. Skipping %s.', env, tag); + return callback(null); + } + + listImages(process.env[env], function (error, result) { + if (error) return callback(error); + + var images = result; + superagent.get(releaseUrl).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + // we get it as text + var releases = JSON.parse(result.text); + + cleanupEnvironment(env, tag, images, releases, callback); + }); + }); + }; +} + +if (p.list) { + async.series([ + handleListEnvironment(p.development, 'DIGITAL_OCEAN_TOKEN_DEV', 'Development', 'https://s3.amazonaws.com/dev-cloudron-releases/versions.json'), + handleListEnvironment(p.staging, 'DIGITAL_OCEAN_TOKEN_STAGING', 'Staging', 'https://s3.amazonaws.com/staging-cloudron-releases/versions.json'), + handleListEnvironment(p.production, 'DIGITAL_OCEAN_TOKEN_PROD', 'Production', 'https://s3.amazonaws.com/prod-cloudron-releases/versions.json') + ], function (error) { + if (error) { + console.log(error); + process.exit(1); + } + + process.exit(0); + }); +} + +if (p.cleanup) { + async.series([ + handleCleanupEnvironment(p.development, 'DIGITAL_OCEAN_TOKEN_DEV', 'Development', 'https://s3.amazonaws.com/dev-cloudron-releases/versions.json'), + handleCleanupEnvironment(p.staging, 'DIGITAL_OCEAN_TOKEN_STAGING', 'Staging', 'https://s3.amazonaws.com/staging-cloudron-releases/versions.json'), + handleCleanupEnvironment(p.production, 'DIGITAL_OCEAN_TOKEN_PROD', 'Production', 'https://s3.amazonaws.com/prod-cloudron-releases/versions.json') + ], function (error) { + if (error) { + console.log(error); + process.exit(1); + } + + process.exit(0); + }); +} diff --git a/release/parsechangelog.js b/release/parsechangelog.js new file mode 100644 index 000000000..19d9c6509 --- /dev/null +++ b/release/parsechangelog.js @@ -0,0 +1,31 @@ +'use strict'; + +var fs = require('fs'); + +exports = module.exports = { + parse: parse +}; + +function parse(version) { + var changelog = [ ]; + var lines = fs.readFileSync(__dirname + '/CHANGES', 'utf8').split('\n'); + for (var i = 0; i < lines.length; i++) { + if (lines[i] === '[' + version + ']') break; + } + + for (i = i + 1; i < lines.length; i++) { + if (lines[i] === '') continue; + if (lines[i][0] === '[') break; + + lines[i] = lines[i].trim(); + + // detect and remove list style - and * in changelog lines + if (lines[i].indexOf('-') === 0) lines[i] = lines[i].slice(1).trim(); + if (lines[i].indexOf('*') === 0) lines[i] = lines[i].slice(1).trim(); + + changelog.push(lines[i]); + } + + return changelog; +} + diff --git a/release/release b/release/release new file mode 100755 index 000000000..38ffcd9e2 --- /dev/null +++ b/release/release @@ -0,0 +1,591 @@ +#!/usr/bin/env node + +'use strict'; + +require('supererror')({ splatchError: true }); +require('colors'); + +var superagent = require('superagent'), + async = require('async'), + safe = require('safetydance'), + AWS = require('aws-sdk'), + yesno = require('yesno'), + Table = require('easy-table'), + program = require('commander'), + semver = require('semver'), + util = require('util'), + versionsFormat = require('./versionsformat.js'), + execSync = require('child_process').execSync, + parseChangelog = require('./parsechangelog.js').parse, + url = require('url'), + path = require('path'), + postmark = require('postmark')(process.env.POSTMARK_API_KEY_TOOLS), + assert = require('assert'); + +var DIGITALOCEAN = 'https://api.digitalocean.com/v2'; + +var ENVIRONMENTS = { + 'dev': { + tag: 'dev', + url: 'https://s3.amazonaws.com/dev-cloudron-releases/versions.json', + accessKeyId: process.env.AWS_DEV_ACCESS_KEY, + secretAccessKey: process.env.AWS_DEV_SECRET_KEY, + releasesBucket: 'dev-cloudron-releases', + digitalOceanToken: process.env.DIGITAL_OCEAN_TOKEN_DEV + }, + 'staging': { + tag: 'staging', + url: 'https://s3.amazonaws.com/staging-cloudron-releases/versions.json', + accessKeyId: process.env.AWS_STAGING_ACCESS_KEY, + secretAccessKey: process.env.AWS_STAGING_SECRET_KEY, + releasesBucket: 'staging-cloudron-releases', + digitalOceanToken: process.env.DIGITAL_OCEAN_TOKEN_STAGING + }, + 'prod': { + tag: 'prod', + url: 'https://s3.amazonaws.com/prod-cloudron-releases/versions.json', + accessKeyId: process.env.AWS_PROD_ACCESS_KEY, + secretAccessKey: process.env.AWS_PROD_SECRET_KEY, + releasesBucket: 'prod-cloudron-releases', + digitalOceanToken: process.env.DIGITAL_OCEAN_TOKEN_PROD + } +}; + +function exit(error) { + if (error) console.error(error.message ? error.message.red : error); + + process.exit(error ? 1 : 0); +} + +function notifyAdmins(env, releases, callback) { + console.log('Notifying admins about new release'.gray); + + var sortedVersions = Object.keys(releases).sort(semver.compare); + var oldVersion = sortedVersions[sortedVersions.length - 2], + newVersion = sortedVersions[sortedVersions.length - 1]; + + var oldImageRef = releases[oldVersion].imageName.match('box-(prod|staging|dev)-([0-9a-z.]+)-.*')[2], + newImageRef = releases[newVersion].imageName.match('box-(prod|staging|dev)-([0-9a-z.]+)-.*')[2]; + + var imageLogs = execSync(util.format('git log %s..%s --format=oneline', oldImageRef, newImageRef), { cwd: __dirname }).toString('utf8'), + imageStat = execSync(util.format('git diff --stat %s..%s', oldImageRef, newImageRef), { cwd: __dirname }).toString('utf8'); + + var oldBoxRef = url.parse(releases[oldVersion].sourceTarballUrl).path.match('/box-(.*).tar.gz')[1], + newBoxRef = url.parse(releases[newVersion].sourceTarballUrl).path.match('/box-(.*).tar.gz')[1]; + + var boxRepo = path.resolve(__dirname, '../../box'); + + var boxLogs = execSync(util.format('git log %s..%s --format=oneline', oldBoxRef, newBoxRef), { cwd: boxRepo }).toString('utf8'), + boxStat = execSync(util.format('git diff --stat %s..%s', oldBoxRef, newBoxRef), { cwd: boxRepo }).toString('utf8'); + + var textBody = util.format( + 'A new box release was pushed by %s.\n\n' + + 'Image Changes\n' + + '-----------------\n' + + '%s\n\n%s\n\n' + + 'Box Changes\n' + + '-----------\n' + + '%s\n\n%s\n\n' + + 'Changelog\n' + + '---------\n' + + '%s\n\n' + + 'Release json\n' + + '------------\n' + + '%s\n\n' + + 'Regards,\n' + + 'Release team\n', + releases[newVersion].author, imageLogs, imageStat, boxLogs, boxStat, + releases[newVersion].changelog, JSON.stringify(releases[newVersion], null, 4)); + + postmark.send({ + 'From': 'no-reply@cloudron.io', + 'To': 'admin@cloudron.io', + 'Subject': util.format('[%s] New box release %s', env.tag, newVersion), + 'TextBody': textBody, + 'Tag': 'Important' + }, callback); +} + +function verifyAndUpload(env, releases, callback) { + assert.strictEqual(typeof env, 'object'); + assert.strictEqual(typeof releases, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var s3 = new AWS.S3({ + accessKeyId: env.accessKeyId, + secretAccessKey: env.secretAccessKey + }); + + var error = versionsFormat.verify(releases); + if (error) return callback(error); + + s3.putObject({ + Bucket: env.releasesBucket, + Key: 'versions.json', + ACL: 'public-read', + Body: JSON.stringify(releases, null, 4), + ContentType: 'application/json' + }, function (error, data) { + if (error) return callback(error); + + console.log('Uploaded'.green); + + callback(null); + }); +} + +function newRelease(options) { + var env = ENVIRONMENTS[options.env]; + if (!env) exit(new Error(util.format('Unknown environment %s', options.env))); + + if (!options.file) exit(new Error('--file is required')); + + var contents = safe.fs.readFileSync(options.file, 'utf8'); + if (!contents) exit(safe.error); + + var releases = safe.JSON.parse(contents); + if (!releases) exit(new Error(options.file + ' has invalid json :' + safe.error.message)); + + verifyAndUpload(env, releases, exit); +} + +function createRelease(options) { + var env = ENVIRONMENTS[options.env]; + if (!env) exit(new Error(util.format('Unknown environment %s', options.env))); + + if (env.tag === 'prod') { + if (options.revert || options.rerelease || options.revert) return exit(new Error('operation is not allowed in prod')); + } + + if (!options.rerelease && !options.revert) { + if (!options.code && !options.image) exit(new Error('--code or --image is required')); + } + + if (options.image && !parseInt(options.image, 10)) exit('image must be a number'); + if (options.code && !safe.url.parse(options.code)) exit('code must be a valid url'); + + var username = execSync('git config user.name').toString('utf8').trim(); + var email = execSync('git config user.email').toString('utf8').trim(); + + superagent.get(env.url).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + var releases = result.type === 'application/json' ? result.body : safe.JSON.parse(result.text); + + if (!releases) exit(new Error('versions.json is not valid JSON')); + + var sortedVersions = Object.keys(releases).sort(semver.rcompare); + var lastVersion = sortedVersions[0]; + + if (options.revert) { + var secondLastVersion = sortedVersions[1]; + + releases[secondLastVersion].next = null; + delete releases[lastVersion]; + + console.log('Reverting %s'.gray, lastVersion); + return verifyAndUpload(env, releases, exit); + } + + var newVersion = options.amend ? lastVersion : semver.inc(lastVersion, 'patch'); + releases[lastVersion].next = newVersion; + + var newImageId = options.image ? parseInt(options.image, 10) : releases[lastVersion].imageId; + var sourceTarballUrl = options.code || releases[lastVersion].sourceTarballUrl; + var upgrade = options.upgrade || (releases[lastVersion].imageId !== newImageId); + + // check if we have a changelog otherwise + var changelog = parseChangelog(newVersion); + if (changelog.length === 0) console.log('No changelog for version %s found.'.yellow, newVersion.bold); + + var url = DIGITALOCEAN + '/images/' + newImageId; + superagent.get(url).set('Authorization', 'Bearer ' + env.digitalOceanToken).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + releases[newVersion] = { + sourceTarballUrl: sourceTarballUrl, + imageId: newImageId, + imageName: result.body.image.name, + changelog: changelog, + upgrade: upgrade, + date: (new Date()).toString(), + author: username + ' <' + email + '>', + next: null + }; + + verifyAndUpload(env, releases, function (error) { + if (error) return exit(error); + + console.log('%s : %s', newVersion, JSON.stringify(releases[newVersion], null, 4)); + + exit(); + }); + }); + }); +} + +function listRelease(options) { + var env = ENVIRONMENTS[options.env]; + if (!env) exit(new Error(util.format('Unknown environment %s', options.env))); + + var raw = !!options.raw; + + superagent.get(env.url).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + if (raw) { + console.log(JSON.stringify(result.body, null, 4)); + exit(null); + } + + console.log(''); + console.log('%s:'.gray, env.tag); + console.log(''); + + if (result.type !== 'application/json') { + console.log('The content type of the release file is %s. It should be application/json something might have gone wrong!'.red, result.type); + console.log('Trying to parse it anyway...'); + console.log(''); + result.body = safe.JSON.parse(result.text); + if (!result.body) { + console.log('Release file is not valid JSON!'.red); + exit(); + } + } + + if (Object.keys(result.body).length === 0) { + console.log('No releases'); + exit(null); + } + + var t = new Table(); + + for (var release in result.body) { + t.cell('Release', release); + t.cell('Image ID', result.body[release].imageId + (result.body[release].upgrade ? '*' : '')); + t.cell('Image Name', result.body[release].imageName); + t.cell('Date', result.body[release].date); + t.cell('Author', result.body[release].author); + t.cell('Next', result.body[release].next); + t.cell('Source', result.body[release].sourceTarballUrl.slice(result.body[release].sourceTarballUrl.lastIndexOf('/') + 1)); + t.newRow(); + } + + console.log(t.toString()); + + exit(null); + }); +} + +function touchRelease(options, callback) { + var env = ENVIRONMENTS[options.env]; + if (!env) exit(new Error(util.format('Unknown environment %s', options.env))); + + superagent.get(env.url).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + var latestVersion = Object.keys(result.body).sort(semver.rcompare)[0]; + result.body[latestVersion].date = (new Date()).toString(); + + verifyAndUpload(env, result.body, exit); + }); +} + +function listImages(token, callback) { + var images = []; + var nextPage = DIGITALOCEAN + '/images?private=true'; + + async.doWhilst(function (callback) { + superagent.get(nextPage).set('Authorization', 'Bearer ' + token).end(function (error, result) { + if (error || result.error) return callback(error || result.error); + + nextPage = (result.body.links && result.body.links.pages && nextPage !== result.body.links.pages.next) ? result.body.links.pages.next : null; + images = images.concat(result.body.images); + + callback(null); + }); + }, function () { return !!nextPage; }, function (error) { + if (error) return callback(error); + callback(null, images); + }); +} + +function sync(options) { + var destEnv = ENVIRONMENTS[options.env]; + if (!destEnv) exit(new Error(util.format('Unknown environment %s', options.env))); + + var sourceEnv; + + if (destEnv.tag === 'staging') sourceEnv = ENVIRONMENTS['prod']; + else if (destEnv.tag === 'dev') sourceEnv = ENVIRONMENTS['staging']; + else exit('Unable to determine source environment to sync from'); + + console.log('Syncing %s to %s', sourceEnv.tag.cyan.bold, destEnv.tag.cyan.bold); + + var S3 = new AWS.S3({ + accessKeyId: destEnv.accessKeyId, + secretAccessKey: destEnv.secretAccessKey + }); + + superagent.get(sourceEnv.url).end(function (error, result) { + if (error || result.error) exit(error || result.error); + + var sourceReleases = result.body; + var destReleases = {}; + + var params = { + Bucket: destEnv.releasesBucket, + Prefix: 'box-' + }; + + S3.listObjects(params, function(error, data) { + if (error) exit(error); + + var devSourceTarballs = data.Contents; + + listImages(destEnv.digitalOceanToken, function (error, images) { + if (error) exit(error); + + for (var release in sourceReleases) { + var match = sourceReleases[release].imageName.match(/box-(?:prod|staging|dev)-(.*)-\d\d\d\d-\d\d-\d\d/); + if (!match || !match[1]) exit('Unable to parse image name %s of release %s.', sourceReleases[release].imageName, release); + + var sourceImageRevision = match[1]; + + // find a suitable image and sourceTarballUrl on dev + var suitableImage = null; + var suitableSourceTarball = null; + + images.forEach(function (image) { + if (image.name.indexOf(util.format('box-%s-%s', destEnv.tag, sourceImageRevision)) === 0) { + suitableImage = image; + } + }); + + devSourceTarballs.forEach(function (tarball) { + if (sourceReleases[release].sourceTarballUrl.indexOf(tarball.Key) !== -1) { + suitableSourceTarball = 'https://' + destEnv.releasesBucket + '.s3.amazonaws.com/' + tarball.Key; + } + }); + + if (!suitableImage) { + console.log('Unable to find a suitable image on %s for release %s.', destEnv.tag, release); + console.log('Required image revision is %s', sourceImageRevision); + process.exit(1); + } + + if (!suitableSourceTarball) { + console.log('Unable to find a suitable source tarball on %s for release %s.', destEnv.tag, release); + console.log('Required source tarball is %s', sourceReleases[release].sourceTarballUrl.slice(sourceReleases[release].sourceTarballUrl.lastIndexOf('/') + 1)); + process.exit(1); + } + + destReleases[release] = { + sourceTarballUrl: suitableSourceTarball, + imageId: suitableImage.id, + imageName: suitableImage.name, + changelog: sourceReleases[release].changelog, + upgrade: sourceReleases[release].upgrade, + date: sourceReleases[release].date, + author: sourceReleases[release].author, + next: sourceReleases[release].next + }; + } + + console.log('Potential %s release file:', destEnv.tag); + console.log(''); + console.log(destReleases); + console.log(''); + + yesno.ask('Do you want to upload that release file? [y/N]', false, function (ok) { + if (!ok) process.exit(1); + + var params = { + Bucket: destEnv.releasesBucket, + Key: 'versions.json', + ACL: 'public-read', + Body: JSON.stringify(destReleases, null, 4), + ContentType: 'application/json' + }; + + S3.putObject(params, function(error, data) { + if (error) { + console.error(error); + process.exit(1); + } + + console.log('Upload successful.'); + process.exit(0); + }); + }); + }); + }); + }); +} + +function getImageByRevision(env, revision, callback) { + assert.strictEqual(typeof revision, 'string'); + assert.strictEqual(typeof callback, 'function'); + + var url = DIGITALOCEAN + '/images?per_page=100'; + superagent.get(url).set('Authorization', 'Bearer ' + env.digitalOceanToken).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + var images = result.body.images; + for (var i = 0; i < images.length; i++) { + if (images[i].name.indexOf('box-' + env.tag + '-' + revision) === 0) return callback(null, images[i]); + } + + callback(new Error('No image for ' + revision)); + }); +} + +function stage(fromEnv, toEnv) { + var username = execSync('git config user.name').toString('utf8').trim(); + var email = execSync('git config user.email').toString('utf8').trim(); + + console.log('Staging from %s -> %s'.gray, fromEnv.tag, toEnv.tag); + + superagent.get(fromEnv.url).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + var fromReleases = result.type === 'application/json' ? result.body : safe.JSON.parse(result.text); + if (!fromReleases) exit(new Error('versions.json is not valid JSON')); + + superagent.get(toEnv.url).end(function (error, result) { + if (error || result.error) return exit(error || result.error); + + var toReleases = result.type === 'application/json' ? result.body : safe.JSON.parse(result.text); + if (!toReleases) exit(new Error('versions.json is not valid JSON')); + + var latestFromVersion = Object.keys(fromReleases).sort(semver.rcompare)[0]; + var latestToVersion = Object.keys(toReleases).sort(semver.rcompare)[0]; + var nextVersion = semver.inc(latestToVersion, 'patch'); + + console.log('Releasing version %s to %s'.gray, nextVersion, toEnv.tag); + + // check if we even have a new version to stage + if (latestFromVersion === latestToVersion) exit(util.format('No new version on %s to stage.', fromEnv.tag)); + + // check if we have a changelog + var changelog = parseChangelog(nextVersion); + if (changelog.length === 0) exit(new Error('No changelog found for version ' + nextVersion)); + + var latestFromImageName = fromReleases[latestFromVersion].imageName; + var latestFromImageRevision = new RegExp('box-' + fromEnv.tag + '-([a-z,0-9.]+)-.*').exec(latestFromImageName)[1]; + + if (!latestFromImageRevision) exit('Unable to determine image revision'); + + getImageByRevision(toEnv, latestFromImageRevision, function (error, toImage) { + if (error) return exit(error); + + var sourceTarballName = url.parse(fromReleases[latestFromVersion].sourceTarballUrl).pathname.substr(1); + var upgrade = toReleases[latestToVersion].imageId !== toImage.id; + + console.log('Copying source code tarball %s to %s'.gray, sourceTarballName, toEnv.tag); + + var cmd = util.format( + 's3cmd get -v --ssl --access_key="%s" --secret_key="%s" "s3://%s/%s" - ' + + ' | s3cmd put -v --ssl --add-header=x-amz-acl:authenticated-read --access_key="%s" --secret_key="%s" - "s3://%s/%s"', + fromEnv.accessKeyId, fromEnv.secretAccessKey, fromEnv.releasesBucket, sourceTarballName, + toEnv.accessKeyId, toEnv.secretAccessKey, toEnv.releasesBucket, sourceTarballName + ); + + execSync(cmd, { stdio: [ null, process.stdout, process.stderr ] } ); + + toReleases[latestToVersion].next = nextVersion; + toReleases[nextVersion] = { + imageId: toImage.id, + imageName: toImage.name, + changelog: changelog, + upgrade: upgrade, + date: (new Date()).toString(), + sourceTarballUrl: 'https://' + toEnv.releasesBucket + '.s3.amazonaws.com/' + sourceTarballName, + author: username + ' <' + email + '>', + next: null + }; + + verifyAndUpload(toEnv, toReleases, function (error) { + if (error) return exit(error); + + console.log('%s : %s', nextVersion, JSON.stringify(toReleases[nextVersion], null, 4)); + + notifyAdmins(toEnv, toReleases, exit); + }); + }); + }); + }); +} + +program.version('0.0.1'); + +program.command('create') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .option('--code ', 'Source code url') + .option('--image ', 'Image id') + .option('--changelog ', 'Changelog') + .option('--upgrade', 'Set the upgrade flag') + .description('Create a new release') + .action(createRelease); + +program.command('revert') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .description('Revert the last release. Use with care') + .action(function (options) { options.revert = true; createRelease(options); }); + +program.command('new') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .option('--file ', 'Upload file as versions.json') + .description('Upload a new versions.json') + .action(newRelease); + +program.command('amend') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .option('--code ', 'Source code url') + .option('--image ', 'Image id') + .option('--changelog ', 'Changelog') + .option('--upgrade', 'Set the upgrade flag') + .description('Amend last release. Use with care') + .action(function (options) { options.amend = true; createRelease(options); }); + +program.command('rerelease') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .description('Make a new release, same as the last release') + .action(function (options) { options.rerelease = true; createRelease(options); }); + +program.command('list') + .option('--raw', 'Show raw json') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .description('List the releases file') + .action(listRelease); + +program.command('sync') + .option('--env ', 'Environment (dev/staging)', 'dev') + .description('Sync the specified env with the parent env (prod -> staging or staging -> dev)') + .action(sync); + +program.command('stage') + .description('Stage latest dev version to staging') + .action(stage.bind(null, ENVIRONMENTS['dev'], ENVIRONMENTS['staging'])); + +program.command('touch') + .option('--env ', 'Environment (dev/staging/prod)', 'dev') + .description('Touch the releases file') + .action(touchRelease); + +program.command('publish') + .description('Publish latest staging version to production') + .action(stage.bind(null, ENVIRONMENTS['staging'], ENVIRONMENTS['prod'])); + +program.parse(process.argv); + +if (!process.argv.slice(2).length) { + program.outputHelp(); +} else { // https://github.com/tj/commander.js/issues/338 + var knownCommand = program.commands.some(function (command) { return command._name === process.argv[2]; }); + if (!knownCommand) { + console.error('Unknown command: ' + process.argv[2]); + process.exit(1); + } +} + diff --git a/release/versions.json b/release/versions.json new file mode 100644 index 000000000..b12a84030 --- /dev/null +++ b/release/versions.json @@ -0,0 +1,418 @@ +{ + "0.0.1": { + "sourceTarballUrl": "https://s3.amazonaws.com/cloudron-releases/box-5b369d2b78605140be63c8c2dc3e4af1ea6ae17b.tar.gz", + "imageId": 10504128, + "imageName": "box-e5d4524-2015-02-06-172850", + "changelog": [ + "Hot Chocolate" + ], + "upgrade": false, + "date": "Fri Feb 6 17:25:45 UTC 2015", + "next": "0.0.2" + }, + "0.0.2": { + "sourceTarballUrl": "https://s3.amazonaws.com/cloudron-releases/box-f2b6340c32c29e5e265abcd7044433d68ac0024c.tar.gz", + "imageId": 10504128, + "imageName": "box-e5d4524-2015-02-06-172850", + "changelog": [ + "Hotfix appstore ui in webadim" + ], + "upgrade": false, + "date": "Fri Feb 6 19:13:26 UTC 2015", + "next": "0.0.3" + }, + "0.0.3": { + "sourceTarballUrl": "https://s3.amazonaws.com/staging-cloudron-releases/box-20e43bdf9c6cf40d3412c59750bc43e834ec39d3.tar.gz", + "imageId": 10621904, + "imageName": "box-8c16ea0-2015-02-12-154005", + "changelog": [ + "Tall Pike" + ], + "upgrade": true, + "date": "Fri Feb 13 00:40:22 UTC 2015", + "next": "0.0.4" + }, + "0.0.4": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-20e43bdf9c6cf40d3412c59750bc43e834ec39d3.tar.gz", + "imageId": 10624164, + "imageName": "box-0ec7efa-2015-02-12-181028", + "changelog": [ + "Ghost release" + ], + "upgrade": false, + "date": "Tue Feb 17 18:03:31 UTC 2015", + "next": "0.0.5" + }, + "0.0.5": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-94383e98591934b648713eccfa67a3f7bbaf659b.tar.gz", + "imageId": 10694830, + "imageName": "box-24cfd4d-2015-02-18-140547", + "changelog": [ + "Banana Smoothie" + ], + "upgrade": true, + "date": "Thu Feb 19 00:13:35 UTC 2015", + "next": "0.0.6" + }, + "0.0.6": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-00dbdddce752d454a6a37b3b15eff9f24b0d8882.tar.gz", + "imageId": 10787693, + "imageName": "box-d7e153f-2015-02-25-192418", + "changelog": [ + "Chai Latte" + ], + "upgrade": true, + "date": "Thu Feb 26 04:19:48 UTC 2015", + "next": "0.0.7" + }, + "0.0.7": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-00dbdddce752d454a6a37b3b15eff9f24b0d8882.tar.gz", + "imageId": 10787693, + "imageName": "box-d7e153f-2015-02-25-192418", + "changelog": [ + "Rerelease for updating SSL certificates" + ], + "upgrade": true, + "date": "Fri Feb 27 07:49:36 UTC 2015", + "next": "0.0.8" + }, + "0.0.8": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-0b5b4535de027a0abccd823b75b4937ec4926d6c.tar.gz", + "imageId": 10881993, + "imageName": "box-3ad90f0-2015-03-04-155817", + "changelog": [ + "Orange Pekoe", + "It's all coming together!" + ], + "upgrade": true, + "date": "Thu Mar 5 00:23:34 UTC 2015", + "next": "0.0.9" + }, + "0.0.9": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-bbe982f14f861b39636ab37072d4e3b3c44a55ac.tar.gz", + "imageId": 10881993, + "imageName": "box-3ad90f0-2015-03-04-155817", + "changelog": [ + "Kashayam" + ], + "upgrade": false, + "date": "Mon Mar 9 23:21:42 UTC 2015", + "next": "0.0.10" + }, + "0.0.10": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-5aac5bd56fe8b917b198bdf7ec4b4bbe231e292c.tar.gz", + "imageId": 10881993, + "imageName": "box-3ad90f0-2015-03-04-155817", + "changelog": [ + "Hot fix for GitLab" + ], + "upgrade": false, + "date": "Tue Mar 10 02:40:53 UTC 2015", + "next": "0.0.11" + }, + "0.0.11": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-b50692fc670ef6ab1bd35d14076cf20fa48cf002.tar.gz", + "imageId": 10881993, + "imageName": "box-3ad90f0-2015-03-04-155817", + "changelog": [ + "Fix app updates" + ], + "upgrade": false, + "date": "Tue Mar 10 18:44:55 UTC 2015", + "next": "0.0.12" + }, + "0.0.12": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-062f94b335e8b57caf9ec4780402f023297fc1b7.tar.gz", + "imageId": 11055383, + "imageName": "box-4e04584-2015-03-17-161439", + "changelog": [ + "Port binding fixes" + ], + "upgrade": true, + "date": "Tue Mar 17 23:35:01 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.13" + }, + "0.0.13": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-ab0f9f691192c735825bb8aa04c1d246b22b067b.tar.gz", + "imageId": 11055383, + "imageName": "box-4e04584-2015-03-17-161439", + "changelog": [ + "Implement App ids" + ], + "upgrade": false, + "date": "Mon Mar 23 02:50:53 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.14" + }, + "0.0.14": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-4cabe44e998be028a86293827c58685f66ae2412.tar.gz", + "imageId": 11055383, + "imageName": "box-4e04584-2015-03-17-161439", + "changelog": [ + "Fix App updates" + ], + "upgrade": false, + "date": "Mon Mar 23 04:50:42 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.15" + }, + "0.0.15": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-d19af1ff6aa47cf2053b9e279078e0421579be57.tar.gz", + "imageId": 11055383, + "imageName": "box-4e04584-2015-03-17-161439", + "changelog": [ + "Fix manifest format" + ], + "upgrade": false, + "date": "Mon Mar 23 05:53:05 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.16" + }, + "0.0.16": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-d19af1ff6aa47cf2053b9e279078e0421579be57.tar.gz", + "imageId": 11162711, + "imageName": "box-e34c6ce-2015-03-25-121127", + "changelog": [ + "Image upgrade with newer addons" + ], + "upgrade": false, + "date": "Wed Mar 25 19:33:45 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.17" + }, + "0.0.17": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-5c1fb62adb35fc311565eb6495dc2985cfc6dc3d.tar.gz", + "imageId": 11162711, + "imageName": "box-e34c6ce-2015-03-25-121127", + "changelog": [ + "Subdomain API changes" + ], + "upgrade": false, + "date": "Wed Mar 25 19:41:28 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.18" + }, + "0.0.18": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-d609c0b3052422813a03a68403c4def47f2ffcba.tar.gz", + "imageId": 11162711, + "imageName": "box-e34c6ce-2015-03-25-121127", + "changelog": [ + "Reorg app data" + ], + "upgrade": false, + "date": "Wed Mar 25 20:41:25 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.19" + }, + "0.0.19": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-9b38e9a6f2e2abad725fedd4f78323619661ed55.tar.gz", + "imageId": 11162711, + "imageName": "box-e34c6ce-2015-03-25-121127", + "changelog": [ + "Delegate app dir creation" + ], + "upgrade": false, + "date": "Thu Mar 26 03:10:53 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.20" + }, + "0.0.20": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-b1381263d3e243eb986d4d9cbf7d2853171941bf.tar.gz", + "imageId": 11162711, + "imageName": "box-e34c6ce-2015-03-25-121127", + "changelog": [ + "Change backup strategy" + ], + "upgrade": false, + "date": "Wed Apr 1 08:18:50 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.21" + }, + "0.0.21": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-0db08cf3efc8345f90d41203cc18f747f42499fa.tar.gz", + "imageId": 11162711, + "imageName": "box-e34c6ce-2015-03-25-121127", + "changelog": [ + "Remove hacks for backup migration" + ], + "upgrade": false, + "date": "Wed Apr 1 08:38:52 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.22" + }, + "0.0.22": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-9cbaef5b41df6a9f5c597448427fe706d2fb0221.tar.gz", + "imageId": 11282641, + "imageName": "box-28a9001-2015-04-02-193801", + "changelog": [ + "Backup fixes" + ], + "upgrade": true, + "date": "Fri Apr 3 02:58:35 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.23" + }, + "0.0.23": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-038ae04f9edbe5727931eace1a62b083c6cebd79.tar.gz", + "imageId": 11384175, + "imageName": "box-1c48a4b-2015-04-09-231714", + "changelog": [ + "Polish, polish, polish" + ], + "upgrade": true, + "date": "Fri Apr 10 06:46:56 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.24" + }, + "0.0.24": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-a168846a056e03f5804cc7b0de8bf9438aa0a4a5.tar.gz", + "imageId": 11390303, + "imageName": "box-94f1086-2015-04-10-174805", + "changelog": [ + "Upgrade all apps and containers" + ], + "upgrade": true, + "date": "Sat Apr 11 01:10:31 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.25" + }, + "0.0.25": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-0a82fcb15f060aa53c1ec0f767d3e56f707416aa.tar.gz", + "imageId": 11390303, + "imageName": "box-94f1086-2015-04-10-174805", + "changelog": [ + "Fix backup creation issue caused by execFile buffer overflow" + ], + "upgrade": false, + "date": "Mon Apr 13 05:00:04 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.26" + }, + "0.0.26": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-d8ac0330e8c9821c5094d7ba985cbece963489c1.tar.gz", + "imageId": 11390303, + "imageName": "box-94f1086-2015-04-10-174805", + "changelog": [ + "Update manifestformat" + ], + "upgrade": false, + "date": "Mon Apr 13 19:54:17 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.27" + }, + "0.0.27": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-c9b7d9a0181911fce6cbc79f5cc965affa2ab2ae.tar.gz", + "imageId": 11390303, + "imageName": "box-94f1086-2015-04-10-174805", + "changelog": [ + "_docker", + "Rickshaw inclusion" + ], + "upgrade": false, + "date": "Fri Apr 17 16:37:25 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.28" + }, + "0.0.28": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-c9b7d9a0181911fce6cbc79f5cc965affa2ab2ae.tar.gz", + "imageId": 11463691, + "imageName": "box-f620aed-2015-04-17-090956", + "changelog": [ + "docker image cleanup" + ], + "upgrade": true, + "date": "Fri Apr 17 16:40:53 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.29" + }, + "0.0.29": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-b22136f11b0b7f51f718aad18904725a6f5c95db.tar.gz", + "imageId": 11463691, + "imageName": "box-f620aed-2015-04-17-090956", + "changelog": [ + "Fix app states" + ], + "upgrade": false, + "date": "Tue Apr 21 17:27:00 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.30" + }, + "0.0.30": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-73de1a7c4e422e4f320b8e8c336e06b48eee1241.tar.gz", + "imageId": 11535902, + "imageName": "box-abaa2f6-2015-04-22-115250", + "changelog": [ + "Fix nginx and collectd configuration setup", + "Foundation for updating box without app restart" + ], + "upgrade": true, + "date": "Wed Apr 22 21:44:00 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.31" + }, + "0.0.31": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-a36d6d864f8905706b94bf7f0c3e50ae0ae857f4.tar.gz", + "imageId": 11535902, + "imageName": "box-abaa2f6-2015-04-22-115250", + "changelog": [ + "Do not restart apps for box update" + ], + "upgrade": false, + "date": "Thu Apr 23 02:51:43 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.32" + }, + "0.0.32": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-69f9bee6777db74416530aba09541243d215194e.tar.gz", + "imageId": 11535902, + "imageName": "box-abaa2f6-2015-04-22-115250", + "changelog": [ + "Fix nginx templating", + "Implement infrastructure versioning" + ], + "upgrade": false, + "date": "Fri Apr 24 18:21:42 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.33" + }, + "0.0.33": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-11aecac6fde56f2289d711e4dee60baa1ccf4d00.tar.gz", + "imageId": 11567620, + "imageName": "box-e877472-2015-04-24-160626", + "changelog": [ + "Fix bug in retire", + "Fix non-infra upgrade" + ], + "upgrade": true, + "date": "Sat Apr 25 01:43:44 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.34" + }, + "0.0.34": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-88bbe86000066eb264c2fa00a55d3b9ddd3e50dd.tar.gz", + "imageId": 11567620, + "imageName": "box-e877472-2015-04-24-160626", + "changelog": [ + "Fix addons ownership", + "Fix backup for upgrades" + ], + "upgrade": false, + "date": "Sat Apr 25 03:34:48 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": "0.0.35" + }, + "0.0.35": { + "sourceTarballUrl": "https://staging-cloudron-releases.s3.amazonaws.com/box-7b7df404b7e7463853d440ba07becceac49c1888.tar.gz", + "imageId": 11876727, + "imageName": "box-cc1f57c-2015-05-15-200507", + "changelog": [ + "Change restart policy of containers to always", + "WARNING! This is an upgrade your cloudron will restart" + ], + "upgrade": true, + "date": "Fri May 15 17:17:40 UTC 2015", + "author": "Girish Ramakrishnan ", + "next": null + } +} diff --git a/release/versionsformat.js b/release/versionsformat.js new file mode 100644 index 000000000..290907018 --- /dev/null +++ b/release/versionsformat.js @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +'use strict'; + +var fs = require('fs'), + safe = require('safetydance'), + semver = require('semver'), + util = require('util'), + url = require('url'); + +exports = module.exports = { + verifyFile: verifyFile, + verify: verify +}; + +function verify(versionsJson) { + if (!versionsJson || typeof versionsJson !== 'object') return new Error('versions must be valid object'); + + // check all the keys + var sortedVersions = Object.keys(versionsJson).sort(semver.compare); + for (var i = 0; i < sortedVersions.length; i++) { + var version = sortedVersions[i]; + if (typeof versionsJson[version].imageId !== 'number') return new Error('version ' + version + ' does not have proper imageId'); + + if (typeof versionsJson[version].imageName !== 'string' || !versionsJson[version].imageName.length) return new Error('version ' + version + ' does not have proper imageName'); + + if ('changeLog' in versionsJson[version] && !util.isArray(versionsJson[version].changeLog)) return new Error('version ' + version + ' does not have proper changeLog'); + + if (typeof versionsJson[version].date !== 'string' || ((new Date(versionsJson[version].date)).toString() === 'Invalid Date')) return new Error('invalid date or missing date'); + + if (versionsJson[version].next !== null && typeof versionsJson[version].next !== 'string') return new Error('version ' + version + ' does not have proper next'); + + if (typeof versionsJson[version].sourceTarballUrl !== 'string') return new Error('version ' + version + ' does not have proper sourceTarballUrl'); + + if ('author' in versionsJson[version] && typeof versionsJson[version].author !== 'string') return new Error('author must be a string'); + + var tarballUrl = url.parse(versionsJson[version].sourceTarballUrl); + if (tarballUrl.protocol !== 'https:') return new Error('sourceTarballUrl must be https'); + if (!/.tar.gz$/.test(tarballUrl.path)) return new Error('sourceTarballUrl must be tar.gz'); + + var nextVersion = versionsJson[version].next; + // despite having the 'next' field, the appstore code currently relies on all versions being sorted based on semver.compare (see boxversions.js) + if (nextVersion && semver.gt(version, nextVersion)) return new Error('next version cannot be less than current @' + version); + } + + // check that package.json version is in versions.json + var currentVersion = require('../package.json').version; + if (sortedVersions.indexOf(currentVersion) === -1) { + return new Error('package.json version is not present in versions.json'); + } + + return null; +} + +function verifyFile(versionsFileName) { + // check if the json is valid + var versions = safe.JSON.parse(fs.readFileSync(versionsFileName)); + if (!versions) { + return new Error(versionsFileName + ' is not valid json : ' + safe.error); + } + + return verify(versions); +} + + diff --git a/src/announce.js b/src/announce.js new file mode 100644 index 000000000..7eea6a32c --- /dev/null +++ b/src/announce.js @@ -0,0 +1,65 @@ +/* jslint node: true */ + +'use strict'; + +var assert = require('assert'), + debug = require('debug')('installer:announce'), + fs = require('fs'), + os = require('os'), + superagent = require('superagent'); + +exports = module.exports = { + start: start, + stop: stop +}; + +var gAnnounceTimerId = null; + +var ANNOUNCE_INTERVAL = parseInt(process.env.ANNOUNCE_INTERVAL, 10) || 60000; // exported for testing + +function start(apiServerOrigin, callback) { + assert.strictEqual(typeof apiServerOrigin, 'string'); + assert.strictEqual(typeof callback, 'function'); + + if (fs.existsSync('/home/yellowtent/box')) { + debug('already provisioned, skipping announce'); + return callback(null); + } + + debug('started'); + + gAnnounceTimerId = setInterval(doAnnounce.bind(null, apiServerOrigin), ANNOUNCE_INTERVAL); + doAnnounce(apiServerOrigin); + + callback(null); +} + +function stop(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Stopping announce'); + + clearInterval(gAnnounceTimerId); + gAnnounceTimerId = null; + + callback(null); +} + +function doAnnounce(apiServerOrigin) { + // On Digital Ocean, the only value which we can give a new droplet is the hostname. + // We use that value to identify the droplet by the appstore server when the droplet + // announce itself. This identifier can look different for other box providers. + var hostname = os.hostname(); + var url = apiServerOrigin + '/api/v1/boxes/' + hostname + '/announce'; + debug('box with %s.', url); + + superagent.get(url).timeout(10000).end(function (error, result) { + if (error || result.statusCode !== 200) { + debug('unable to announce to app server, try again.', error); + return; + } + + debug('success'); + }); +} + diff --git a/src/installer.js b/src/installer.js new file mode 100644 index 000000000..f2a2bffe1 --- /dev/null +++ b/src/installer.js @@ -0,0 +1,100 @@ +/* jslint node: true */ + +'use strict'; + +var assert = require('assert'), + child_process = require('child_process'), + debug = require('debug')('installer:installer'), + path = require('path'), + util = require('util'); + +exports = module.exports = { + InstallerError: InstallerError, + + provision: provision, + restore: restore, + update: update, + retire: retire +}; + +var INSTALLER_CMD = path.join(__dirname, 'scripts/installer.sh'), + RETIRE_CMD = path.join(__dirname, 'scripts/retire.sh'), + SUDO = '/usr/bin/sudo'; + +function InstallerError(reason, info) { + Error.call(this); + Error.captureStackTrace(this, this.constructor); + + this.name = this.constructor.name; + this.reason = reason; + this.message = !info ? reason : (typeof info === 'object' ? JSON.stringify(info) : info); +} +util.inherits(InstallerError, Error); +InstallerError.INTERNAL_ERROR = 1; +InstallerError.ALREADY_PROVISIONED = 2; + +function update(args, callback) { + provision(args, callback); +} + +function restore(args, callback) { + provision(args, callback); +} + +function spawn(tag, cmd, args, callback) { + assert.strictEqual(typeof tag, 'string'); + assert.strictEqual(typeof cmd, 'string'); + assert(util.isArray(args)); + assert.strictEqual(typeof callback, 'function'); + + var cp = child_process.spawn(cmd, args, { timeout: 0 }); + cp.stdout.setEncoding('utf8'); + cp.stdout.on('data', function (data) { debug('%s (stdout): %s', tag, data); }); + cp.stderr.setEncoding('utf8'); + cp.stderr.on('data', function (data) { debug('%s (stderr): %s', tag, data); }); + + cp.on('error', function (error) { + debug('%s : child process errored %s', tag, error.message); + callback(error); + }); + + cp.on('exit', function (code, signal) { + debug('%s : child process exited. code: %d signal: %d', tag, code, signal); + if (signal) return callback(new Error('Exited with signal ' + signal)); + if (code !== 0) return callback(new Error('Exited with code ' + code)); + + callback(null); + }); +} + +function retire(args, callback) { + assert.strictEqual(typeof args, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var pargs = [ RETIRE_CMD ]; + pargs.push('--data', JSON.stringify(args.data)); + + debug('retire: calling with args %j', pargs); + + if (process.env.NODE_ENV === 'test') return callback(null); + + // sudo is required for retire() + spawn('retire', SUDO, pargs, callback); +} + +function provision(args, callback) { + assert.strictEqual(typeof args, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var pargs = [ INSTALLER_CMD ]; + pargs.push('--sourcetarballurl', args.sourceTarballUrl); + pargs.push('--data', JSON.stringify(args.data)); + + debug('provision: calling with args %j', pargs); + + if (process.env.NODE_ENV === 'test') return callback(null); + + // sudo is required for update() + spawn('provision', SUDO, pargs, callback); +} + diff --git a/src/scripts/installer.sh b/src/scripts/installer.sh new file mode 100755 index 000000000..26286b8f4 --- /dev/null +++ b/src/scripts/installer.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -eu -o pipefail + +readonly BOX_SRC_DIR=/home/yellowtent/box +readonly DATA_DIR=/home/yellowtent/data + +readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly json="${script_dir}/../../node_modules/.bin/json" +readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400" + +readonly is_update=$([[ -d "${BOX_SRC_DIR}" ]] && echo "yes" || echo "no") + +# create a provision file for testing. %q escapes args. %q is reused as much as necessary to satisfy $@ +(echo -e "#!/bin/bash\n"; printf "%q " "${script_dir}/installer.sh" "$@") > /home/yellowtent/provision.sh +chmod +x /home/yellowtent/provision.sh + +arg_source_tarball_url="" +arg_data="" + +args=$(getopt -o "" -l "sourcetarballurl:,data:" -n "$0" -- "$@") +eval set -- "${args}" + +while true; do + case "$1" in + --sourcetarballurl) arg_source_tarball_url="$2";; + --data) arg_data="$2";; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac + + shift 2 +done + +box_src_tmp_dir=$(mktemp -dt box-src-XXXXXX) +echo "Downloading box code from ${arg_source_tarball_url} to ${box_src_tmp_dir}" + +while true; do + if $curl -L "${arg_source_tarball_url}" | tar -zxf - -C "${box_src_tmp_dir}"; then break; fi + echo "Failed to download source tarball, trying again" + sleep 5 +done +(cd "${box_src_tmp_dir}" && npm rebuild) + +if [[ "${is_update}" == "yes" ]]; then + echo "Setting up update splash screen" + "${box_src_tmp_dir}/setup/splashpage.sh" --data "${arg_data}" # show splash from new code + ${BOX_SRC_DIR}/setup/stop.sh # stop the old code +fi + +# switch the codes +rm -rf "${BOX_SRC_DIR}" +mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}" +chown -R yellowtent.yellowtent "${BOX_SRC_DIR}" + +# create a start file for testing. %q escapes args +(echo -e "#!/bin/bash\n"; printf "%q " "${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}") > /home/yellowtent/setup_start.sh +chmod +x /home/yellowtent/setup_start.sh + +echo "Calling box setup script" +"${BOX_SRC_DIR}/setup/start.sh" --data "${arg_data}" + diff --git a/src/scripts/retire.sh b/src/scripts/retire.sh new file mode 100755 index 000000000..1d8363eca --- /dev/null +++ b/src/scripts/retire.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# This script is called once at the end of a cloudrons lifetime + +set -eu -o pipefail + +readonly BOX_SRC_DIR=/home/yellowtent/box + +arg_data="" + +args=$(getopt -o "" -l "data:" -n "$0" -- "$@") +eval set -- "${args}" + +while true; do + case "$1" in + --data) arg_data="$2";; + --) break;; + *) echo "Unknown option $1"; exit 1;; + esac + + shift 2 +done + +echo "Setting up splash screen" +"${BOX_SRC_DIR}/setup/splashpage.sh" --retire --data "${arg_data}" # show splash +"${BOX_SRC_DIR}/setup/stop.sh" # stop the cloudron code diff --git a/src/server.js b/src/server.js new file mode 100755 index 000000000..f232ab02d --- /dev/null +++ b/src/server.js @@ -0,0 +1,293 @@ +#!/usr/bin/env node + +/* jslint node: true */ + +'use strict'; + +var announce = require('./announce.js'), + assert = require('assert'), + async = require('async'), + debug = require('debug')('installer:server'), + express = require('express'), + fs = require('fs'), + http = require('http'), + HttpError = require('connect-lastmile').HttpError, + https = require('https'), + HttpSuccess = require('connect-lastmile').HttpSuccess, + installer = require('./installer.js'), + json = require('body-parser').json, + lastMile = require('connect-lastmile'), + morgan = require('morgan'), + path = require('path'), + safe = require('safetydance'), + superagent = require('superagent'), + ts = require('tail-stream'); + +exports = module.exports = { + start: start, + stop: stop +}; + +var gHttpsServer = null, // provision server; used for install/restore + gHttpServer = null; // update server; used for updates + +// 'data' is opaque. the following code exists to help debugging +function checkData(data) { + assert.strictEqual(typeof data, 'object'); + + if (typeof data.token !== 'string') console.error('No token provided'); + if (typeof data.apiServerOrigin !== 'string') console.error('No apiServerOrigin provided'); + if (typeof data.webServerOrigin !== 'string') console.error('No webServerOrigin provided'); + if (typeof data.fqdn !== 'string') console.error('No fqdn provided'); + if (typeof data.tlsCert !== 'string') console.error('No TLS cert provided'); + if (typeof data.tlsKey !== 'string') console.error('No TLS key provided'); + if (typeof data.isCustomDomain !== 'boolean') console.error('No isCustomDomain provided'); + if (typeof data.version !== 'string') console.error('No version provided'); + if (typeof data.sourceTarballUrl !== 'string') console.error('No sourceTarballUrl provided'); + + if ('restoreUrl' in data) { + if (typeof data.restoreUrl !== 'string') console.error('No restoreUrl provided'); + if (typeof data.restoreKey !== 'string') console.error('No restoreKey provided'); + } +} + +function restore(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided')); + + if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided')); + + checkData(req.body.data); + + debug('restore: received from appstore %j', req.body); + + installer.restore(req.body, function (error) { + if (error) console.error(error); + }); + + announce.stop(function () { }); + + next(new HttpSuccess(202, { })); +} + +function provision(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided')); + + if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided')); + + checkData(req.body.data); + + debug('provision: received from appstore %j', req.body); + + installer.provision(req.body, function (error) { + if (error) console.error(error); + }); + + announce.stop(function () { }); + + next(new HttpSuccess(202, { })); +} + +function update(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (typeof req.body.sourceTarballUrl !== 'string') return next(new HttpError(400, 'No sourceTarballUrl provided')); + + if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided')); + + checkData(req.body.data); + + debug('update: started'); + + installer.update(req.body, function (error) { + if (error) console.error(error); + }); + + next(new HttpSuccess(202, { })); +} + +function retire(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (!req.body.data || typeof req.body.data !== 'object') return next(new HttpError(400, 'No data provided')); + + if (typeof req.body.data.tlsCert !== 'string') console.error('No TLS cert provided'); + if (typeof req.body.data.tlsKey !== 'string') console.error('No TLS key provided'); + + debug('retire: received from appstore %j', req.body); + + installer.retire(req.body, function (error) { + if (error) console.error(error); + }); + + next(new HttpSuccess(202, {})); +} + +function logs(req, res, next) { + if (!req.query.filename) return next(new HttpError(400, 'No filename provided')); + var tail = req.query.tail === 'true'; + var stream = null; + + var stat = safe.fs.statSync(req.query.filename); + + if (!stat) return res.status(404).send('Not found'); + + if (tail) { + var tailStreamOptions = { + beginAt: 'end', + onMove: 'follow', + detectTruncate: true, + onTruncate: 'end', + endOnError: true + }; + + stream = safe(function () { return ts.createReadStream(req.query.filename, tailStreamOptions); }); + stream.destroy = stream.end; // tail-stream closes it's watchers with this special API + } else { + stream = fs.createReadStream(req.query.filename); + res.set('content-length', stat.size); + } + + if (!stream) return res.status(404).send(safe.error.message); + + stream.on('error', function (error) { res.write(error.message); res.end(); }); + res.on('close', function () { stream.destroy(); }); + res.status(200); + stream.pipe(res); +} + +function backup(req, res, next) { + // !! below port has to be in sync with box/config.js internalPort + superagent.post('http://127.0.0.1:3001/api/v1/backup').end(function (error, result) { + if (error) return next(new HttpError(500, error)); + if (result.statusCode !== 202) return next(new HttpError(500, 'trigger backup failed with ' + result.statusCode)); + next(new HttpSuccess(202, {})); + }); +} + +function startUpdateServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Starting update server'); + + var app = express(); + + var router = new express.Router(); + + if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false })); + + app.use(json({ strict: true })) + .use(router) + .use(lastMile()); + + router.post('/api/v1/installer/update', update); + + gHttpServer = http.createServer(app); + gHttpServer.on('error', console.error); + + gHttpServer.listen(2020, '127.0.0.1', callback); +} + +function startProvisionServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Starting provision server'); + + var app = express(); + + var router = new express.Router(); + + if (process.env.NODE_ENV !== 'test') app.use(morgan('dev', { immediate: false })); + + app.use(json({ strict: true })) + .use(router) + .use(lastMile()); + + router.post('/api/v1/installer/provision', provision); + router.post('/api/v1/installer/restore', restore); + router.post('/api/v1/installer/retire', retire); + router.get ('/api/v1/installer/logs', logs); + router.post('/api/v1/installer/backup', backup); + router.post('/api/v1/installer/update', update); + + var caPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? '../../keys/installer_ca' : 'certs'); + var certPath = path.join(__dirname, process.env.NODE_ENV === 'test' ? '../../keys/installer' : 'certs'); + + var options = { + key: fs.readFileSync(path.join(certPath, 'server.key')), + cert: fs.readFileSync(path.join(certPath, 'server.crt')), + ca: fs.readFileSync(path.join(caPath, 'ca.crt')), + + // request cert from client and only allow from our CA + requestCert: true, + rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0' // this is set in the tests + }; + + gHttpsServer = https.createServer(options, app); + gHttpsServer.on('error', console.error); + + gHttpsServer.listen(process.env.NODE_ENV === 'test' ? 4443 : 886, '0.0.0.0', callback); +} + +function stopProvisionServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Stopping provision server'); + + if (!gHttpsServer) return callback(null); + + gHttpsServer.close(callback); + gHttpsServer = null; +} + +function stopUpdateServer(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('Stopping update server'); + + if (!gHttpServer) return callback(null); + + gHttpServer.close(callback); + gHttpServer = null; +} + +function start(callback) { + assert.strictEqual(typeof callback, 'function'); + + debug('starting'); + + superagent.get('http://169.254.169.254/metadata/v1.json').end(function (error, result) { + if (error || result.statusCode !== 200) { + console.error('Error getting metadata', error); + return; + } + + var apiServerOrigin = JSON.parse(result.body.user_data).apiServerOrigin; + debug('Using apiServerOrigin from metadata: %s', apiServerOrigin); + + async.series([ + announce.start.bind(null, apiServerOrigin), + startUpdateServer, + startProvisionServer + ], callback); + }); +} + +function stop(callback) { + assert.strictEqual(typeof callback, 'function'); + + async.series([ + announce.stop, + stopUpdateServer, + stopProvisionServer + ], callback); +} + +if (require.main === module) { + start(function (error) { + if (error) console.error(error); + }); +} diff --git a/src/test/installer-test.js b/src/test/installer-test.js new file mode 100644 index 000000000..b53b7e40e --- /dev/null +++ b/src/test/installer-test.js @@ -0,0 +1,332 @@ +/* jslint node:true */ +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +'use strict'; + +var expect = require('expect.js'), + fs = require('fs'), + path = require('path'), + nock = require('nock'), + os = require('os'), + request = require('superagent'), + server = require('../server.js'), + _ = require('lodash'); + +var EXTERNAL_SERVER_URL = 'https://localhost:4443'; +var INTERNAL_SERVER_URL = 'http://localhost:2020'; +var APPSERVER_ORIGIN = 'http://appserver'; +var FQDN = os.hostname(); + +describe('Server', function () { + this.timeout(5000); + + before(function (done) { + var user_data = JSON.stringify({ apiServerOrigin: APPSERVER_ORIGIN }); // user_data is a string + var scope = nock('http://169.254.169.254') + .persist() + .get('/metadata/v1.json') + .reply(200, JSON.stringify({ user_data: user_data }), { 'Content-Type': 'application/json' }); + done(); + }); + + after(function (done) { + nock.cleanAll(); + done(); + }); + + describe('starts and stop', function () { + it('starts', function (done) { + server.start(done); + }); + + it('stops', function (done) { + server.stop(done); + }); + }); + + describe('update (internal server)', function () { + before(function (done) { + server.start(done); + }); + after(function (done) { + server.stop(done); + }); + + it('does not respond to provision', function (done) { + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/provision').send({ }).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(404); + done(); + }); + }); + + it('does not respond to restore', function (done) { + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/restore').send({ }).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(404); + done(); + }); + }); + + var data = { + sourceTarballUrl: "https://foo.tar.gz", + + data: { + token: 'sometoken', + apiServerOrigin: APPSERVER_ORIGIN, + webServerOrigin: 'https://somethingelse.com', + fqdn: 'www.something.com', + tlsKey: 'key', + tlsCert: 'cert', + boxVersionsUrl: 'https://versions.json', + version: '0.1' + } + }; + + Object.keys(data).forEach(function (key) { + it('fails due to missing ' + key, function (done) { + var dataCopy = _.merge({ }, data); + delete dataCopy[key]; + + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(dataCopy).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + }); + + it('succeeds', function (done) { + request.post(INTERNAL_SERVER_URL + '/api/v1/installer/update').send(data).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(202); + done(); + }); + }); + }); + + describe('provision - announce', function () { + var failingGet = null; + + before(function (done) { + process.env.ANNOUNCE_INTERVAL = 20; + + var scope = nock(APPSERVER_ORIGIN); + failingGet = scope.get('/api/v1/boxes/' + FQDN + '/announce'); + failingGet.times(5).reply(502); + + server.start(done); + }); + + after(function (done) { + process.env.ANNOUNCE_INTERVAL = 60000; + // failingGet.removeInterceptor({ hostname: 'appserver' }); + server.stop(done); + }); + + it('sends announce request repeatedly', function (done) { + setTimeout(function () { + expect(failingGet.counter).to.be.below(6); // counter is nock update + done(); + }, 100); + }); + }); + + describe('provision - restore', function () { + var data = { + sourceTarballUrl: 'https://sourceTarballUrl', + + data: { + boxVersionsUrl: 'https://versions.json', + version: '0.1', + restoreUrl: 'https://restoreurl', + restoreKey: 'somebackupkey', + token: 'sometoken', + apiServerOrigin: APPSERVER_ORIGIN, + webServerOrigin: 'https://somethingelse.com', + fqdn: 'www.something.com', + tlsKey: 'key', + tlsCert: 'cert' + } + }; + + before(function (done) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead + server.start(done); + }); + + after(function (done) { + server.stop(done); + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + }); + + Object.keys(data).forEach(function (key) { + it('fails due to missing ' + key, function (done) { + var dataCopy = _.merge({ }, data); + delete dataCopy[key]; + + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/restore').send(dataCopy).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + }); + + it('succeeds', function (done) { + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/restore').send(data).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(202); + done(); + }); + }); + }); + + describe('provision - provision', function () { + var data = { + sourceTarballUrl: 'https://sourceTarballUrl', + + data: { + boxVersionsUrl: 'https://versions.json', + version: '0.1', + token: 'sometoken', + apiServerOrigin: APPSERVER_ORIGIN, + webServerOrigin: 'https://somethingelse.com', + fqdn: 'www.something.com', + tlsKey: 'key', + tlsCert: 'cert' + } + }; + + before(function (done) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead + server.start(done); + }); + + after(function (done) { + server.stop(done); + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + }); + + Object.keys(data).forEach(function (key) { + it('fails due to missing ' + key, function (done) { + var dataCopy = _.merge({ }, data); + delete dataCopy[key]; + + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/provision').send(dataCopy).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + }); + + it('succeeds', function (done) { + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/provision').send(data).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(202); + done(); + }); + }); + }); + + describe('logs', function () { + before(function (done) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead + server.start(done); + }); + + after(function (done) { + server.stop(done); + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + }); + + it('needs filename', function (done) { + request.get(EXTERNAL_SERVER_URL + '/api/v1/installer/logs').end(function (error, result) { + expect(!error).to.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + + it('returns stream for valid file', function (done) { + request.get(EXTERNAL_SERVER_URL + '/api/v1/installer/logs?filename=' + __filename).end(function (error, result) { + expect(!error).to.be.ok(); + expect(result.headers['content-length']).to.be('' + fs.statSync(__filename).size); + expect(result.statusCode).to.equal(200); + done(); + }); + }); + + it('returns tail stream for valid file', function (done) { + var tailFile = path.join(os.tmpdir(), 'test-tail'); + fs.writeFileSync(tailFile, 'line 1\n'); + + var res = request.get(EXTERNAL_SERVER_URL + '/api/v1/installer/logs?tail=true&filename=' + tailFile).end(function (error, result) { + expect(!error).to.be.ok(); + expect(result.headers['transfer-encoding']).to.be('chunked'); + expect(result.statusCode).to.equal(200); + + fs.unlinkSync(tailFile); + + done(); + }); + + // push some new log lines to trigger request.get() callback + setTimeout(function () { fs.appendFileSync(tailFile, 'line 2\n'); }, 100); + setTimeout(res.abort.bind(res), 200); + }); + + it('returns 404 for missing file', function (done) { + request.get(EXTERNAL_SERVER_URL + '/api/v1/installer/logs?filename=/tmp/randomtotally').end(function (error, result) { + expect(!error).to.be.ok(); + expect(result.statusCode).to.equal(404); + done(); + }); + }); + }); + + describe('retire', function () { + var data = { + data: { + tlsKey: 'key', + tlsCert: 'cert' + } + }; + + before(function (done) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; // TODO: use a installer ca signed cert instead + server.start(done); + }); + + after(function (done) { + server.stop(done); + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + }); + + Object.keys(data).forEach(function (key) { + it('fails due to missing ' + key, function (done) { + var dataCopy = _.merge({ }, data); + delete dataCopy[key]; + + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(dataCopy).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(400); + done(); + }); + }); + }); + + it('succeeds', function (done) { + request.post(EXTERNAL_SERVER_URL + '/api/v1/installer/retire').send(data).end(function (error, result) { + expect(error).to.not.be.ok(); + expect(result.statusCode).to.equal(202); + done(); + }); + }); + }); +}); +