From 72c19c994037217fea89195a552098f79ceafb70 Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Wed, 15 Jul 2015 18:17:37 -0700 Subject: [PATCH] Rework backup and update code --- src/appdb.js | 3 +- src/apps.js | 141 +++++++++++++++++++++++++--- src/apptask.js | 28 +++++- src/backups.js | 202 ++--------------------------------------- src/cloudron.js | 113 ++++++++++++++++++++--- src/cron.js | 11 ++- src/locker.js | 13 ++- src/routes/apps.js | 24 +---- src/routes/backups.js | 2 + src/routes/cloudron.js | 6 +- src/taskmanager.js | 6 +- src/updater.js | 15 +-- 12 files changed, 302 insertions(+), 262 deletions(-) diff --git a/src/appdb.js b/src/appdb.js index d4537d364..16990f633 100644 --- a/src/appdb.js +++ b/src/appdb.js @@ -30,6 +30,7 @@ exports = module.exports = { ISTATE_PENDING_UNINSTALL: 'pending_uninstall', ISTATE_PENDING_RESTORE: 'pending_restore', ISTATE_PENDING_UPDATE: 'pending_update', + ISTATE_PENDING_BACKUP: 'pending_backup', ISTATE_ERROR: 'error', ISTATE_INSTALLED: 'installed', @@ -339,7 +340,7 @@ function setInstallationCommand(appId, installationState, values, callback) { updateWithConstraints(appId, values, '', callback); } else if (installationState === exports.ISTATE_PENDING_RESTORE) { updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback); - } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE) { + } else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) { updateWithConstraints(appId, values, 'AND installationState = "installed"', callback); } else { callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState')); diff --git a/src/apps.js b/src/apps.js index 3f0b40755..9ac07d37d 100644 --- a/src/apps.js +++ b/src/apps.js @@ -12,9 +12,15 @@ exports = module.exports = { install: install, configure: configure, uninstall: uninstall, + restore: restore, + restoreApp: restoreApp, + update: update, + backup: backup, + backupApp: backupApp, + getLogStream: getLogStream, getLogs: getLogs, @@ -34,9 +40,12 @@ exports = module.exports = { _validatePortBindings: validatePortBindings }; -var appdb = require('./appdb.js'), +var addons = require('./addons.js'), + appdb = require('./appdb.js'), assert = require('assert'), async = require('async'), + backups = require('./backups.js'), + BackupsError = require('./backups.js').BackupsError, config = require('../config.js'), constants = require('../constants.js'), DatabaseError = require('./databaseerror.js'), @@ -48,13 +57,32 @@ var appdb = require('./appdb.js'), paths = require('./paths.js'), safe = require('safetydance'), semver = require('semver'), + shell = require('./shell.js'), split = require('split'), superagent = require('superagent'), taskmanager = require('./taskmanager.js'), util = require('util'), validator = require('validator'); -var NOOP_CALLBACK = function (error) { console.error(error); }; +var NOOP_CALLBACK = function (error) { console.error(error); }, + BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'), + BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'); + +function debugApp(app, args) { + assert(!app || typeof app === 'object'); + + var prefix = app ? app.location : '(no app)'; + debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1))); +} + +function ignoreError(func) { + return function (callback) { + func(function (error) { + if (error) console.error('Ignored error:', error); + callback(); + }); + }; +} // http://dustinsenos.com/articles/customErrorsInNode // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi @@ -78,6 +106,7 @@ function AppsError(reason, errorOrMessage) { } util.inherits(AppsError, Error); AppsError.INTERNAL_ERROR = 'Internal Error'; +AppsError.EXTERNAL_ERROR = 'External Error'; AppsError.ALREADY_EXISTS = 'Already Exists'; AppsError.NOT_FOUND = 'Not Found'; AppsError.BAD_FIELD = 'Bad Field'; @@ -610,19 +639,19 @@ function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) { }); } -function canAutoupdateApp(app, newManifest) { - // TODO: maybe check the description as well? - for (var env in newManifest.tcpPorts) { - if (!(env in app.portBindings)) return false; - } - - return true; -} - function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } } assert.strictEqual(typeof updateInfo, 'object'); assert.strictEqual(typeof callback, 'function'); + function canAutoupdateApp(app, newManifest) { + // TODO: maybe check the description as well? + for (var env in newManifest.tcpPorts) { + if (!(env in app.portBindings)) return false; + } + + return true; + } + if (!updateInfo) return callback(null); async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) { @@ -640,3 +669,93 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma }, callback); } +function backupApp(app, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof callback, 'function'); + + function canBackupApp(app) { + // only backup apps that are installed or pending configure. Rest of them are in some + // state not good for consistent backup + + return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) + || app.installationState === appdb.ISTATE_PENDING_CONFIGURE + || app.installationState === appdb.ISTATE_PENDING_BACKUP + || app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask + } + + if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy')); + + var appConfig = { + manifest: app.manifest, + location: app.location, + portBindings: app.portBindings, + accessRestriction: app.accessRestriction + }; + + if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) { + return callback(safe.error); + } + + backups.getBackupUrl(app, null, function (error, result) { + if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message)); + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id); + + async.series([ + ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])), + addons.backupAddons.bind(null, app, app.manifest), + shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD, app.id, result.url, result.backupKey ]), + ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])), + ], function (error) { + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + debugApp(app, 'backupApp: successful id:%s', result.id); + + setRestorePoint(app.id, result.id, appConfig, function (error) { + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + return callback(null, result.id); + }); + }); + }); +} + +function backup(appId, callback) { + assert.strictEqual(typeof appId, 'string'); + assert.strictEqual(typeof callback, 'function'); + + get(appId, function (error, app) { + if (error && error.reason === AppsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND)); + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_BACKUP, values, function (error) { + if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + taskmanager.restartAppTask(appId); + + callback(null); + }); + }); +} + +function restoreApp(app, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof callback, 'function'); + assert(app.lastBackupId); + + backups.getRestoreUrl(app.lastBackupId, function (error, result) { + if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message)); + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + debugApp(app, 'restoreApp: restoreUrl:%s', result.url); + + shell.sudo('restoreApp', [ RESTORE_APP_CMD, app.id, result.url, result.backupKey ], function (error) { + if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error)); + + addons.restoreAddons(app, app.manifest, callback); + }); + }); +} + diff --git a/src/apptask.js b/src/apptask.js index 5ff2341a9..99b94d75a 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -30,7 +30,6 @@ var addons = require('./addons.js'), apps = require('./apps.js'), assert = require('assert'), async = require('async'), - backups = require('./backups.js'), clientdb = require('./clientdb.js'), config = require('../config.js'), database = require('./database.js'), @@ -573,6 +572,25 @@ function install(app, callback) { }); } +function backup(app, callback) { + async.series([ + updateApp.bind(null, app, { installationProgress: '10, Backing up' }), + apps.backupApp.bind(null, app), + + // done! + function (callback) { + debugApp(app, 'installed'); + updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '' }, callback); + } + ], function seriesDone(error) { + if (error) { + debugApp(app, 'error installing app: %s', error); + return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error)); + } + callback(null); + }); +} + // restore is always called with a previous backup. restore is also called for upgrades and infra updates function restore(app, callback) { assert(app.lastBackupId); @@ -614,7 +632,7 @@ function restore(app, callback) { createVolume.bind(null, app), updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }), - backups.restoreApp.bind(null, app), + apps.restoreApp.bind(null, app), updateApp.bind(null, app, { installationProgress: '75, Creating container' }), deleteContainer.bind(null, app), @@ -706,7 +724,7 @@ function update(app, callback) { updateApp.bind(null, app, { installationProgress: '10, Backup app' }), function (done) { - backups.backupApp(app, function (error) { + apps.backupApp(app, function (error) { if (error) error.backupError = true; done(error); }); @@ -852,6 +870,10 @@ function startTask(appId, callback) { return restore(app, callback); } + if (app.installationState === appdb.ISTATE_PENDING_BACKUP) { + return backup(app, callback); + } + if (app.installationState === appdb.ISTATE_INSTALLED) { return handleRunCommand(app, callback); } diff --git a/src/backups.js b/src/backups.js index 54673d81b..e22b504fb 100644 --- a/src/backups.js +++ b/src/backups.js @@ -4,38 +4,17 @@ exports = module.exports = { BackupsError: BackupsError, getAllPaged: getAllPaged, - scheduleAppBackup: scheduleAppBackup, getBackupUrl: getBackupUrl, - getRestoreUrl: getRestoreUrl, - - backup: backup, - backupBox: backupBox, - backupApp: backupApp, - - restoreApp: restoreApp + getRestoreUrl: getRestoreUrl }; -var addons = require('./addons.js'), - appdb = require('./appdb.js'), - apps = require('./apps.js'), - assert = require('assert'), - async = require('async'), +var assert = require('assert'), config = require('../config.js'), debug = require('debug')('box:backups'), - path = require('path'), - paths = require('./paths.js'), - progress = require('./progress.js'), - safe = require('safetydance'), - shell = require('./shell.js'), superagent = require('superagent'), util = require('util'); -var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'), - BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'), - RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'), - BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'); - function BackupsError(reason, errorOrMessage) { assert.strictEqual(typeof reason, 'string'); assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined'); @@ -55,27 +34,9 @@ function BackupsError(reason, errorOrMessage) { } } util.inherits(BackupsError, Error); -BackupsError.NOT_FOUND = 'not found'; -BackupsError.BAD_STATE = 'bad state'; BackupsError.EXTERNAL_ERROR = 'external error'; BackupsError.INTERNAL_ERROR = 'internal error'; -function debugApp(app, args) { - assert(!app || typeof app === 'object'); - - var prefix = app ? app.location : '(no app)'; - debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1))); -} - -function ignoreError(func) { - return function (callback) { - func(function (error) { - if (error) console.error('Ignored error:', error); - callback(); - }); - }; -} - function getAllPaged(page, perPage, callback) { assert.strictEqual(typeof page, 'number'); assert.strictEqual(typeof perPage, 'number'); @@ -93,31 +54,6 @@ function getAllPaged(page, perPage, callback) { }); } -function canBackupApp(app) { - // only backup apps that are installed or pending configure. Rest of them are in some - // state not good for consistent backup - - return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) || app.installationState === appdb.ISTATE_PENDING_CONFIGURE; -} - -function scheduleAppBackup(appId, callback) { - assert.strictEqual(typeof appId, 'string'); - assert.strictEqual(typeof callback, 'function'); - - apps.get(appId, function (error, app) { - if (error && error.reason === AppsError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND)); - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - if (!canBackupApp(app)) return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy')); - - backupApp(app, function (error) { - if (error) console.error('backup failed.', error); - }); - - callback(null); - }); -} - function getBackupUrl(app, appBackupIds, callback) { assert(!app || typeof app === 'object'); assert(!appBackupIds || util.isArray(appBackupIds)); @@ -133,9 +69,9 @@ function getBackupUrl(app, appBackupIds, callback) { }; superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) { - if (error) return callback(new Error('Error getting presigned backup url: ' + error.message)); - - if (result.statusCode !== 201 || !result.body || !result.body.url) return callback(new Error('Error getting presigned backup url : ' + result.statusCode + ' ' + result.text)); + if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error)); + if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text)); + if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response')); return callback(null, result.body); }); @@ -148,134 +84,12 @@ function getRestoreUrl(backupId, callback) { var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl'; superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) { - if (error) return callback(new Error('Error getting presigned download url: ' + error.message)); - - if (result.statusCode !== 201 || !result.body || !result.body.url) return callback(new Error('Error getting presigned download url : ' + result.statusCode + ' ' + result.text)); + if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error)); + if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text)); + if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response')); return callback(null, result.body); }); } -function restoreApp(app, callback) { - assert(app.lastBackupId); - - getRestoreUrl(app.lastBackupId, function (error, result) { - if (error) return callback(error); - - debugApp(app, 'restoreApp: restoreUrl:%s', result.url); - - shell.sudo('restoreApp', [ RESTORE_APP_CMD, app.id, result.url, result.backupKey ], function (error) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - addons.restoreAddons(app, app.manifest, callback); - }); - }); -} - -function backupApp(app, callback) { - var appConfig = { - manifest: app.manifest, - location: app.location, - portBindings: app.portBindings, - accessRestriction: app.accessRestriction - }; - - if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) { - return callback(safe.error); - } - - getBackupUrl(app, null, function (error, result) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id); - - async.series([ - ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])), - addons.backupAddons.bind(null, app, app.manifest), - shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD, app.id, result.url, result.backupKey ]), - ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])), - ], function (error) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - debugApp(app, 'backupApp: successful id:%s', result.id); - - apps.setRestorePoint(app.id, result.id, appConfig, function (error) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - return callback(null, result.id); - }); - }); - }); -} - -function backupBoxWithAppBackupIds(appBackupIds, callback) { - assert(util.isArray(appBackupIds)); - - getBackupUrl(null /* app */, appBackupIds, function (error, result) { - if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); - - debug('backup: url %s', result.url); - - async.series([ - ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])), - shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]), - ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])), - ], function (error) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - debug('backup: successful'); - - callback(null, result.id); - }); - }); -} - -function backupBox(callback) { - apps.getAll(function (error, allApps) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - var appBackupIds = allApps.map(function (app) { return app.lastBackupId; }); - appBackupIds = appBackupIds.filter(function (id) { return id !== null }); // remove apps that were never backed up - - backupBoxWithAppBackupIds(appBackupIds, callback); - }); -} - -function backup(callback) { - callback = callback || function () { }; // callback can be empty for timer triggered backup - - apps.getAll(function (error, allApps) { - if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - - var processed = 0; - var step = 100/(allApps.length+1); - - progress.set(progress.BACKUP, processed, ''); - - async.mapSeries(allApps, function iterator(app, iteratorCallback) { - ++processed; - - if (canBackupApp(app)) { - return backupApp(app, function (error, backupId) { - progress.set(progress.BACKUP, step * processed, app.location); - iteratorCallback(error, backupId); - }); - } - - debugApp(app, 'Skipping backup (istate:%s health%s). Reusing %s', app.installationState, app.health, app.lastBackupId); - progress.set(progress.BACKUP, step * processed, app.location); - - return iteratorCallback(null, app.lastBackupId); - }, function appsBackedUp(error, backupIds) { - if (error) return callback(error); - - backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up - - backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) { - progress.set(progress.BACKUP, 100, ''); - callback(error, restoreKey); - }); - }); - }); -} diff --git a/src/cloudron.js b/src/cloudron.js index e37cdaab5..8d92e9cc3 100644 --- a/src/cloudron.js +++ b/src/cloudron.js @@ -23,7 +23,9 @@ exports = module.exports = { }; var assert = require('assert'), + async = require('async'), backups = require('./backups.js'), + BackupsError = require('./backups.js').BackupsError, clientdb = require('./clientdb.js'), config = require('../config.js'), debug = require('debug')('box:cloudron'), @@ -45,13 +47,31 @@ var assert = require('assert'), util = require('util'); var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'), - REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'); - -var INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update'; + REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'), + BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'), + BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'), + INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update'; var gAddMailDnsRecordsTimerId = null, gCloudronDetails = null; // cached cloudron details like region,size... +function debugApp(app, args) { + assert(!app || typeof app === 'object'); + + var prefix = app ? app.location : '(no app)'; + debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1))); +} + +function ignoreError(func) { + return function (callback) { + func(function (error) { + if (error) console.error('Ignored error:', error); + callback(); + }); + }; +} + + function CloudronError(reason, errorOrMessage) { assert.strictEqual(typeof reason, 'string'); assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined'); @@ -81,7 +101,6 @@ CloudronError.BAD_PASSWORD = 'Bad password'; CloudronError.BAD_NAME = 'Bad name'; CloudronError.BAD_STATE = 'Bad state'; CloudronError.NOT_FOUND = 'Not found'; -CloudronError.NO_UPDATE_AVAILABLE = 'No update available'; function initialize(callback) { assert.strictEqual(typeof callback, 'function'); @@ -342,7 +361,7 @@ function migrate(size, region, callback) { assert.strictEqual(typeof region, 'string'); assert.strictEqual(typeof callback, 'function'); - backups.backup(function (error, restoreKey) { + backup(function (error, restoreKey) { if (error) return callback(error); debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey); @@ -362,11 +381,10 @@ function migrate(size, region, callback) { }); } -function update(callback) { +function update(boxUpdateInfo, callback) { assert.strictEqual(typeof callback, 'function'); - var boxUpdateInfo = updater.getUpdateInfo().box; - if (!boxUpdateInfo) return next(new CloudronError(CloudronError.NO_UPDATE_AVAILABLE)); + if (!boxUpdateInfo) return next(null); var error = locker.lockForBoxUpdate(); if (error) return next(new CloudronError(CloudronError.BAD_STATE, error.message)); @@ -391,7 +409,7 @@ function doUpgrade(boxUpdateInfo, callback) { progress.set(progress.UPDATE, 5, 'Create app and box backup'); - backups.backup(function (error) { + backupBoxAndApps(function (error) { if (error) return callback(error); superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade') @@ -415,7 +433,7 @@ function doUpdate(boxUpdateInfo, callback) { progress.set(progress.UPDATE, 5, 'Create box backup'); - backups.backupBox(function (error) { + backupBox(function (error) { if (error) return callback(error); // fetch a signed sourceTarballUrl @@ -469,7 +487,7 @@ function backup(callback) { var error = locker.lockForFullBackup(); if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message)); - backups.backup(function (error) { + backupBoxAndApps(function (error) { if (error) console.error('backup failed.', error); locker.unlockForFullBackup(); @@ -496,3 +514,76 @@ function ensureBackup(callback) { }); } +function backupBoxWithAppBackupIds(appBackupIds, callback) { + assert(util.isArray(appBackupIds)); + + backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) { + if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message)); + if (error) return callback(new CloudronError.INTERNAL_ERROR, error); + + debug('backup: url %s', result.url); + + async.series([ + ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])), + shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]), + ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])), + ], function (error) { + if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error)); + + debug('backup: successful'); + + callback(null, result.id); + }); + }); +} + +// this function expects you to have a lock +function backupBox(callback) { + apps.getAll(function (error, allApps) { + if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error)); + + var appBackupIds = allApps.map(function (app) { return app.lastBackupId; }); + appBackupIds = appBackupIds.filter(function (id) { return id !== null }); // remove apps that were never backed up + + backupBoxWithAppBackupIds(appBackupIds, callback); + }); +} + +// this function expects you to have a lock +function backupBoxAndApps(callback) { + callback = callback || function () { }; // callback can be empty for timer triggered backup + + apps.getAll(function (error, allApps) { + if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error)); + + var processed = 0; + var step = 100/(allApps.length+1); + + progress.set(progress.BACKUP, processed, ''); + + async.mapSeries(allApps, function iterator(app, iteratorCallback) { + ++processed; + + apps.backupApp(app, function (error, backupId) { + progress.set(progress.BACKUP, step * processed, app.location); + + if (error && error.reason === AppsError.BAD_STATE) { + debugApp(app, 'Skipping backup (istate:%s health%s). Reusing %s', app.installationState, app.health, app.lastBackupId); + backupId = app.lastBackupId; + } + + return iteratorCallback(null, app.lastBackupId); + }); + }, function appsBackedUp(error, backupIds) { + if (error) return callback(error); + + backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up + + backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) { + progress.set(progress.BACKUP, 100, ''); + callback(error, restoreKey); + }); + }); + }); +} + diff --git a/src/cron.js b/src/cron.js index 5aa2e6f36..626fda746 100644 --- a/src/cron.js +++ b/src/cron.js @@ -6,8 +6,8 @@ exports = module.exports = { }; -var assert = require('assert'), - backups = require('./backups.js'), +var apps = require('./apps.js'), + assert = require('assert'), cloudron = require('./cloudron.js'), CronJob = require('cron').CronJob, debug = require('debug')('box:cron'), @@ -98,7 +98,12 @@ function autoupdatePatternChanged(pattern) { cronTime: pattern, onTick: function() { debug('Starting autoupdate'); - updater.autoupdate(); + var updateInfo = updater.getUpdateInfo(); + if (updateInfo.box) { + cloudron.update(updateInfo.box, NOOP_CALLBACK); + } else if (updateInfo.apps) { + apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK); + } }, start: true, timeZone: gBoxUpdateCheckerJob.cronTime.timeZone // hack diff --git a/src/locker.js b/src/locker.js index 458804442..498bf2e92 100644 --- a/src/locker.js +++ b/src/locker.js @@ -5,7 +5,10 @@ exports = module.exports = { unlockForBoxUpdate: unlockForBoxUpdate, lockForFullBackup: lockForFullBackup, - unlockForFullBackup: unlockForFullBackup + unlockForFullBackup: unlockForFullBackup, + + lockForAppTask: lockForAppTask, + unlockForAppTask: unlockForAppTask }; function lockForBoxUpdate() { @@ -24,3 +27,11 @@ function unlockForFullBackup() { return null; } +function lockForAppTask() { + return null; +} + +function unlockForAppTask() { + return null; +} + diff --git a/src/routes/apps.js b/src/routes/apps.js index 2d05e6770..6a79329ed 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -24,8 +24,6 @@ exports = module.exports = { var apps = require('../apps.js'), AppsError = apps.AppsError, assert = require('assert'), - backups = require('../backups.js'), - BackupsError = backups.BackupsError, debug = require('debug')('box:routes/apps'), fs = require('fs'), HttpError = require('connect-lastmile').HttpError, @@ -177,34 +175,18 @@ function restoreApp(req, res, next) { function backupApp(req, res, next) { assert.strictEqual(typeof req.params.id, 'string'); - debug('Restore app id:%s', req.params.id); + debug('Backup app id:%s', req.params.id); - backups.scheduleAppBackup(req.params.id, function (error) { - if (error && error.reason === BackupsError.NOT_FOUND) return next(new HttpError(404, 'No such app')); - if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message)); - if (error && error.reason === BackupsError.EXTERNAL_ERROR) return next(new HttpError(503, error)); - if (error) return next(new HttpError(500, error)); - - next(new HttpSuccess(202, { })); - }); -} - -function restoreApp(req, res, next) { - assert.strictEqual(typeof req.params.id, 'string'); - - debug('Restore app id:%s', req.params.id); - - apps.restore(req.params.id, function (error) { + apps.backup(req.params.id, function (error) { if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app')); - if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message)); if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message)); + if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error)); if (error) return next(new HttpError(500, error)); next(new HttpSuccess(202, { })); }); } - /* * Uninstalls an app * @bodyparam {string} id The id of the app to be uninstalled diff --git a/src/routes/backups.js b/src/routes/backups.js index a3a3246d4..1644f17dc 100644 --- a/src/routes/backups.js +++ b/src/routes/backups.js @@ -16,12 +16,14 @@ var backups = require('../backups.js'), function get(req, res, next) { backups.getAllPaged(1, 5, function (error, result) { if (error) return next(new HttpError(500, error)); + next(new HttpSuccess(200, { backups: result })); }); } function create(req, res, next) { // don't want for backup to complete since this can take long + // progress can be checked up ny polling the progress api call cloudron.backup(function (error) { if (error) debug('Could not backup', error); }); diff --git a/src/routes/cloudron.js b/src/routes/cloudron.js index 9e06ce737..708a37612 100644 --- a/src/routes/cloudron.js +++ b/src/routes/cloudron.js @@ -129,8 +129,10 @@ function getConfig(req, res, next) { } function update(req, res, next) { - cloudron.update(function (error) { - if (error && error.reason === CloudronError.NO_UPDATE_AVAILABLE) return next(new HttpError(422, 'No update available')); + var boxUpdateInfo = updater.getUpdateInfo().box; + if (!boxUpdateInfo) return next(new HttpError(422, 'No update available')); + + cloudron.update(boxUpdateInfo, function (error) { if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message)); if (error) return next(new HttpError(500, error)); diff --git a/src/taskmanager.js b/src/taskmanager.js index 533d9ecda..aa9bdb288 100644 --- a/src/taskmanager.js +++ b/src/taskmanager.js @@ -11,6 +11,7 @@ var appdb = require('./appdb.js'), assert = require('assert'), child_process = require('child_process'), debug = require('debug')('box:taskmanager'), + locker = require('./locker.js'), _ = require('underscore'); var gActiveTasks = { }; @@ -53,7 +54,9 @@ function startAppTask(appId) { assert.strictEqual(typeof appId, 'string'); assert(!(appId in gActiveTasks)); - if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) { + var lockError = locker.lockForAppTask(); // ## FIXME: need to poll when the lock becomes free + + if (lockError || Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) { debug('Reached concurrency limit, queueing task for %s', appId); gPendingTasks.push(appId); return; @@ -66,6 +69,7 @@ function startAppTask(appId) { appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code }, NOOP_CALLBACK); } delete gActiveTasks[appId]; + locker.unlockForAppTask(); if (gPendingTasks.length !== 0) startAppTask(gPendingTasks.shift()); // start another pending task }); } diff --git a/src/updater.js b/src/updater.js index 30b11c4f1..2bcc37af0 100644 --- a/src/updater.js +++ b/src/updater.js @@ -6,21 +6,18 @@ exports = module.exports = { checkAppUpdates: checkAppUpdates, checkBoxUpdate: checkBoxUpdates, - getUpdateInfo: getUpdateInfo, - autoupdate: autoupdate + getUpdateInfo: getUpdateInfo }; var apps = require('./apps.js'), assert = require('assert'), async = require('async'), - cloudron = require('./cloudron.js'), config = require('../config.js'), debug = require('debug')('box:updater'), fs = require('fs'), mailer = require('./mailer.js'), path = require('path'), paths = require('./paths.js'), - progress = require('./progress.js'), safe = require('safetydance'), semver = require('semver'), superagent = require('superagent'), @@ -162,13 +159,3 @@ function checkBoxUpdates() { }); } -function autoupdate() { - // FIXME: box update and app update must not be concurrent. also, there is no way to track completion of updates - // and this we need to one or the other. - if (gBoxUpdateInfo !== null) { - cloudron.update(NOOP_CALLBACK); - } else { - apps.autoupdateApps(gAppUpdateInfo, NOOP_CALLBACK); - } -} -