jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Rework backup and update code

Browse Source
This commit is contained in:
Girish Ramakrishnan
2015-07-15 18:17:37 -07:00
parent 5521779d1c
commit 72c19c9940
12 changed files with 302 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/appdb.js
+2 -1
View File
@@ -30,6 +30,7 @@ exports = module.exports = {
ISTATE_PENDING_UNINSTALL: 'pending_uninstall',
ISTATE_PENDING_RESTORE: 'pending_restore',
ISTATE_PENDING_UPDATE: 'pending_update',
ISTATE_PENDING_BACKUP: 'pending_backup',
ISTATE_ERROR: 'error',
ISTATE_INSTALLED: 'installed',
@@ -339,7 +340,7 @@ function setInstallationCommand(appId, installationState, values, callback) {
updateWithConstraints(appId, values, '', callback);
} else if (installationState === exports.ISTATE_PENDING_RESTORE) {
updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
} else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE) {
} else if (installationState === exports.ISTATE_PENDING_UPDATE || exports.ISTATE_PENDING_CONFIGURE || installationState == exports.ISTATE_PENDING_BACKUP) {
updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
} else {
callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
src/apps.js
+130 -11
View File
@@ -12,9 +12,15 @@ exports = module.exports = {
install: install,
configure: configure,
uninstall: uninstall,
restore: restore,
restoreApp: restoreApp,
update: update,
backup: backup,
backupApp: backupApp,
getLogStream: getLogStream,
getLogs: getLogs,
@@ -34,9 +40,12 @@ exports = module.exports = {
_validatePortBindings: validatePortBindings
};
var appdb = require('./appdb.js'),
var addons = require('./addons.js'),
appdb = require('./appdb.js'),
assert = require('assert'),
async = require('async'),
backups = require('./backups.js'),
BackupsError = require('./backups.js').BackupsError,
config = require('../config.js'),
constants = require('../constants.js'),
DatabaseError = require('./databaseerror.js'),
@@ -48,13 +57,32 @@ var appdb = require('./appdb.js'),
paths = require('./paths.js'),
safe = require('safetydance'),
semver = require('semver'),
shell = require('./shell.js'),
split = require('split'),
superagent = require('superagent'),
taskmanager = require('./taskmanager.js'),
util = require('util'),
validator = require('validator');
var NOOP_CALLBACK = function (error) { console.error(error); };
var NOOP_CALLBACK = function (error) { console.error(error); },
BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
function debugApp(app, args) {
assert(!app || typeof app === 'object');
var prefix = app ? app.location : '(no app)';
debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
}
function ignoreError(func) {
return function (callback) {
func(function (error) {
if (error) console.error('Ignored error:', error);
callback();
});
};
}
// http://dustinsenos.com/articles/customErrorsInNode
// http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
@@ -78,6 +106,7 @@ function AppsError(reason, errorOrMessage) {
}
util.inherits(AppsError, Error);
AppsError.INTERNAL_ERROR = 'Internal Error';
AppsError.EXTERNAL_ERROR = 'External Error';
AppsError.ALREADY_EXISTS = 'Already Exists';
AppsError.NOT_FOUND = 'Not Found';
AppsError.BAD_FIELD = 'Bad Field';
@@ -610,19 +639,19 @@ function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
});
}
function canAutoupdateApp(app, newManifest) {
// TODO: maybe check the description as well?
for (var env in newManifest.tcpPorts) {
if (!(env in app.portBindings)) return false;
}
return true;
}
function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
assert.strictEqual(typeof updateInfo, 'object');
assert.strictEqual(typeof callback, 'function');
function canAutoupdateApp(app, newManifest) {
// TODO: maybe check the description as well?
for (var env in newManifest.tcpPorts) {
if (!(env in app.portBindings)) return false;
}
return true;
}
if (!updateInfo) return callback(null);
async.eachSeries(Object.keys(updateInfo), function iterator(appId, iteratorDone) {
@@ -640,3 +669,93 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
}, callback);
}
function backupApp(app, callback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof callback, 'function');
function canBackupApp(app) {
// only backup apps that are installed or pending configure. Rest of them are in some
// state not good for consistent backup
return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY)
|| app.installationState === appdb.ISTATE_PENDING_CONFIGURE
|| app.installationState === appdb.ISTATE_PENDING_BACKUP
|| app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
}
if (!canBackupApp(app)) return callback(new AppsError(AppsError.BAD_STATE, 'App not healthy'));
var appConfig = {
manifest: app.manifest,
location: app.location,
portBindings: app.portBindings,
accessRestriction: app.accessRestriction
};
if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
return callback(safe.error);
}
backups.getBackupUrl(app, null, function (error, result) {
if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id);
async.series([
ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
addons.backupAddons.bind(null, app, app.manifest),
shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD, app.id, result.url, result.backupKey ]),
ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
], function (error) {
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
debugApp(app, 'backupApp: successful id:%s', result.id);
setRestorePoint(app.id, result.id, appConfig, function (error) {
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
return callback(null, result.id);
});
});
});
}
function backup(appId, callback) {
assert.strictEqual(typeof appId, 'string');
assert.strictEqual(typeof callback, 'function');
get(appId, function (error, app) {
if (error && error.reason === AppsError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_BACKUP, values, function (error) {
if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
taskmanager.restartAppTask(appId);
callback(null);
});
});
}
function restoreApp(app, callback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof callback, 'function');
assert(app.lastBackupId);
backups.getRestoreUrl(app.lastBackupId, function (error, result) {
if (error && error.reason == BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
shell.sudo('restoreApp', [ RESTORE_APP_CMD, app.id, result.url, result.backupKey ], function (error) {
if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
addons.restoreAddons(app, app.manifest, callback);
});
});
}
src/apptask.js
+25 -3
View File
@@ -30,7 +30,6 @@ var addons = require('./addons.js'),
apps = require('./apps.js'),
assert = require('assert'),
async = require('async'),
backups = require('./backups.js'),
clientdb = require('./clientdb.js'),
config = require('../config.js'),
database = require('./database.js'),
@@ -573,6 +572,25 @@ function install(app, callback) {
});
}
function backup(app, callback) {
async.series([
updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
apps.backupApp.bind(null, app),
// done!
function (callback) {
debugApp(app, 'installed');
updateApp(app, { installationState: appdb.ISTATE_INSTALLED, installationProgress: '' }, callback);
}
], function seriesDone(error) {
if (error) {
debugApp(app, 'error installing app: %s', error);
return updateApp(app, { installationState: appdb.ISTATE_ERROR, installationProgress: error.message }, callback.bind(null, error));
}
callback(null);
});
}
// restore is always called with a previous backup. restore is also called for upgrades and infra updates
function restore(app, callback) {
assert(app.lastBackupId);
@@ -614,7 +632,7 @@ function restore(app, callback) {
createVolume.bind(null, app),
updateApp.bind(null, app, { installationProgress: '70, Download backup and restore addons' }),
backups.restoreApp.bind(null, app),
apps.restoreApp.bind(null, app),
updateApp.bind(null, app, { installationProgress: '75, Creating container' }),
deleteContainer.bind(null, app),
@@ -706,7 +724,7 @@ function update(app, callback) {
updateApp.bind(null, app, { installationProgress: '10, Backup app' }),
function (done) {
backups.backupApp(app, function (error) {
apps.backupApp(app, function (error) {
if (error) error.backupError = true;
done(error);
});
@@ -852,6 +870,10 @@ function startTask(appId, callback) {
return restore(app, callback);
}
if (app.installationState === appdb.ISTATE_PENDING_BACKUP) {
return backup(app, callback);
}
if (app.installationState === appdb.ISTATE_INSTALLED) {
return handleRunCommand(app, callback);
}
src/backups.js
+8 -194
View File
@@ -4,38 +4,17 @@ exports = module.exports = {
BackupsError: BackupsError,
getAllPaged: getAllPaged,
scheduleAppBackup: scheduleAppBackup,
getBackupUrl: getBackupUrl,
getRestoreUrl: getRestoreUrl,
backup: backup,
backupBox: backupBox,
backupApp: backupApp,
restoreApp: restoreApp
getRestoreUrl: getRestoreUrl
};
var addons = require('./addons.js'),
appdb = require('./appdb.js'),
apps = require('./apps.js'),
assert = require('assert'),
async = require('async'),
var assert = require('assert'),
config = require('../config.js'),
debug = require('debug')('box:backups'),
path = require('path'),
paths = require('./paths.js'),
progress = require('./progress.js'),
safe = require('safetydance'),
shell = require('./shell.js'),
superagent = require('superagent'),
util = require('util');
var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
function BackupsError(reason, errorOrMessage) {
assert.strictEqual(typeof reason, 'string');
assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -55,27 +34,9 @@ function BackupsError(reason, errorOrMessage) {
}
}
util.inherits(BackupsError, Error);
BackupsError.NOT_FOUND = 'not found';
BackupsError.BAD_STATE = 'bad state';
BackupsError.EXTERNAL_ERROR = 'external error';
BackupsError.INTERNAL_ERROR = 'internal error';
function debugApp(app, args) {
assert(!app || typeof app === 'object');
var prefix = app ? app.location : '(no app)';
debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
}
function ignoreError(func) {
return function (callback) {
func(function (error) {
if (error) console.error('Ignored error:', error);
callback();
});
};
}
function getAllPaged(page, perPage, callback) {
assert.strictEqual(typeof page, 'number');
assert.strictEqual(typeof perPage, 'number');
@@ -93,31 +54,6 @@ function getAllPaged(page, perPage, callback) {
});
}
function canBackupApp(app) {
// only backup apps that are installed or pending configure. Rest of them are in some
// state not good for consistent backup
return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) || app.installationState === appdb.ISTATE_PENDING_CONFIGURE;
}
function scheduleAppBackup(appId, callback) {
assert.strictEqual(typeof appId, 'string');
assert.strictEqual(typeof callback, 'function');
apps.get(appId, function (error, app) {
if (error && error.reason === AppsError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND));
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
if (!canBackupApp(app)) return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy'));
backupApp(app, function (error) {
if (error) console.error('backup failed.', error);
});
callback(null);
});
}
function getBackupUrl(app, appBackupIds, callback) {
assert(!app || typeof app === 'object');
assert(!appBackupIds || util.isArray(appBackupIds));
@@ -133,9 +69,9 @@ function getBackupUrl(app, appBackupIds, callback) {
};
superagent.put(url).query({ token: config.token() }).send(data).end(function (error, result) {
if (error) return callback(new Error('Error getting presigned backup url: ' + error.message));
if (result.statusCode !== 201 || !result.body || !result.body.url) return callback(new Error('Error getting presigned backup url : ' + result.statusCode + ' ' + result.text));
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
return callback(null, result.body);
});
@@ -148,134 +84,12 @@ function getRestoreUrl(backupId, callback) {
var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/restoreurl';
superagent.put(url).query({ token: config.token(), backupId: backupId }).end(function (error, result) {
if (error) return callback(new Error('Error getting presigned download url: ' + error.message));
if (result.statusCode !== 201 || !result.body || !result.body.url) return callback(new Error('Error getting presigned download url : ' + result.statusCode + ' ' + result.text));
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error));
if (result.statusCode !== 201) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, result.text));
if (!result.body || !result.body.url) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Unexpected response'));
return callback(null, result.body);
});
}
function restoreApp(app, callback) {
assert(app.lastBackupId);
getRestoreUrl(app.lastBackupId, function (error, result) {
if (error) return callback(error);
debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
shell.sudo('restoreApp', [ RESTORE_APP_CMD, app.id, result.url, result.backupKey ], function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
addons.restoreAddons(app, app.manifest, callback);
});
});
}
function backupApp(app, callback) {
var appConfig = {
manifest: app.manifest,
location: app.location,
portBindings: app.portBindings,
accessRestriction: app.accessRestriction
};
if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
return callback(safe.error);
}
getBackupUrl(app, null, function (error, result) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
debugApp(app, 'backupApp: backup url:%s backup id:%s', result.url, result.id);
async.series([
ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
addons.backupAddons.bind(null, app, app.manifest),
shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD, app.id, result.url, result.backupKey ]),
ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
], function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
debugApp(app, 'backupApp: successful id:%s', result.id);
apps.setRestorePoint(app.id, result.id, appConfig, function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
return callback(null, result.id);
});
});
});
}
function backupBoxWithAppBackupIds(appBackupIds, callback) {
assert(util.isArray(appBackupIds));
getBackupUrl(null /* app */, appBackupIds, function (error, result) {
if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
debug('backup: url %s', result.url);
async.series([
ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
], function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
debug('backup: successful');
callback(null, result.id);
});
});
}
function backupBox(callback) {
apps.getAll(function (error, allApps) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
appBackupIds = appBackupIds.filter(function (id) { return id !== null }); // remove apps that were never backed up
backupBoxWithAppBackupIds(appBackupIds, callback);
});
}
function backup(callback) {
callback = callback || function () { }; // callback can be empty for timer triggered backup
apps.getAll(function (error, allApps) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
var processed = 0;
var step = 100/(allApps.length+1);
progress.set(progress.BACKUP, processed, '');
async.mapSeries(allApps, function iterator(app, iteratorCallback) {
++processed;
if (canBackupApp(app)) {
return backupApp(app, function (error, backupId) {
progress.set(progress.BACKUP, step * processed, app.location);
iteratorCallback(error, backupId);
});
}
debugApp(app, 'Skipping backup (istate:%s health%s). Reusing %s', app.installationState, app.health, app.lastBackupId);
progress.set(progress.BACKUP, step * processed, app.location);
return iteratorCallback(null, app.lastBackupId);
}, function appsBackedUp(error, backupIds) {
if (error) return callback(error);
backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
progress.set(progress.BACKUP, 100, '');
callback(error, restoreKey);
});
});
});
}
src/cloudron.js
+102 -11
View File
@@ -23,7 +23,9 @@ exports = module.exports = {
};
var assert = require('assert'),
async = require('async'),
backups = require('./backups.js'),
BackupsError = require('./backups.js').BackupsError,
clientdb = require('./clientdb.js'),
config = require('../config.js'),
debug = require('debug')('box:cloudron'),
@@ -45,13 +47,31 @@ var assert = require('assert'),
util = require('util');
var RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh'),
REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh');
var INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update';
var gAddMailDnsRecordsTimerId = null,
gCloudronDetails = null; // cached cloudron details like region,size...
function debugApp(app, args) {
assert(!app || typeof app === 'object');
var prefix = app ? app.location : '(no app)';
debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
}
function ignoreError(func) {
return function (callback) {
func(function (error) {
if (error) console.error('Ignored error:', error);
callback();
});
};
}
function CloudronError(reason, errorOrMessage) {
assert.strictEqual(typeof reason, 'string');
assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
@@ -81,7 +101,6 @@ CloudronError.BAD_PASSWORD = 'Bad password';
CloudronError.BAD_NAME = 'Bad name';
CloudronError.BAD_STATE = 'Bad state';
CloudronError.NOT_FOUND = 'Not found';
CloudronError.NO_UPDATE_AVAILABLE = 'No update available';
function initialize(callback) {
assert.strictEqual(typeof callback, 'function');
@@ -342,7 +361,7 @@ function migrate(size, region, callback) {
assert.strictEqual(typeof region, 'string');
assert.strictEqual(typeof callback, 'function');
backups.backup(function (error, restoreKey) {
backup(function (error, restoreKey) {
if (error) return callback(error);
debug('migrate: size %s region %s restoreKey %s', size, region, restoreKey);
@@ -362,11 +381,10 @@ function migrate(size, region, callback) {
});
}
function update(callback) {
function update(boxUpdateInfo, callback) {
assert.strictEqual(typeof callback, 'function');
var boxUpdateInfo = updater.getUpdateInfo().box;
if (!boxUpdateInfo) return next(new CloudronError(CloudronError.NO_UPDATE_AVAILABLE));
if (!boxUpdateInfo) return next(null);
var error = locker.lockForBoxUpdate();
if (error) return next(new CloudronError(CloudronError.BAD_STATE, error.message));
@@ -391,7 +409,7 @@ function doUpgrade(boxUpdateInfo, callback) {
progress.set(progress.UPDATE, 5, 'Create app and box backup');
backups.backup(function (error) {
backupBoxAndApps(function (error) {
if (error) return callback(error);
superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
@@ -415,7 +433,7 @@ function doUpdate(boxUpdateInfo, callback) {
progress.set(progress.UPDATE, 5, 'Create box backup');
backups.backupBox(function (error) {
backupBox(function (error) {
if (error) return callback(error);
// fetch a signed sourceTarballUrl
@@ -469,7 +487,7 @@ function backup(callback) {
var error = locker.lockForFullBackup();
if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
backups.backup(function (error) {
backupBoxAndApps(function (error) {
if (error) console.error('backup failed.', error);
locker.unlockForFullBackup();
@@ -496,3 +514,76 @@ function ensureBackup(callback) {
});
}
function backupBoxWithAppBackupIds(appBackupIds, callback) {
assert(util.isArray(appBackupIds));
backups.getBackupUrl(null /* app */, appBackupIds, function (error, result) {
if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new CloudronError(CloudronError.EXTERNAL_ERROR, error.message));
if (error) return callback(new CloudronError.INTERNAL_ERROR, error);
debug('backup: url %s', result.url);
async.series([
ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.url, result.backupKey ]),
ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
], function (error) {
if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
debug('backup: successful');
callback(null, result.id);
});
});
}
// this function expects you to have a lock
function backupBox(callback) {
apps.getAll(function (error, allApps) {
if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
appBackupIds = appBackupIds.filter(function (id) { return id !== null }); // remove apps that were never backed up
backupBoxWithAppBackupIds(appBackupIds, callback);
});
}
// this function expects you to have a lock
function backupBoxAndApps(callback) {
callback = callback || function () { }; // callback can be empty for timer triggered backup
apps.getAll(function (error, allApps) {
if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
var processed = 0;
var step = 100/(allApps.length+1);
progress.set(progress.BACKUP, processed, '');
async.mapSeries(allApps, function iterator(app, iteratorCallback) {
++processed;
apps.backupApp(app, function (error, backupId) {
progress.set(progress.BACKUP, step * processed, app.location);
if (error && error.reason === AppsError.BAD_STATE) {
debugApp(app, 'Skipping backup (istate:%s health%s). Reusing %s', app.installationState, app.health, app.lastBackupId);
backupId = app.lastBackupId;
}
return iteratorCallback(null, app.lastBackupId);
});
}, function appsBackedUp(error, backupIds) {
if (error) return callback(error);
backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
backupBoxWithAppBackupIds(backupIds, function (error, restoreKey) {
progress.set(progress.BACKUP, 100, '');
callback(error, restoreKey);
});
});
});
}
src/cron.js
+8 -3
View File
@@ -6,8 +6,8 @@ exports = module.exports = {
};
var assert = require('assert'),
backups = require('./backups.js'),
var apps = require('./apps.js'),
assert = require('assert'),
cloudron = require('./cloudron.js'),
CronJob = require('cron').CronJob,
debug = require('debug')('box:cron'),
@@ -98,7 +98,12 @@ function autoupdatePatternChanged(pattern) {
cronTime: pattern,
onTick: function() {
debug('Starting autoupdate');
updater.autoupdate();
var updateInfo = updater.getUpdateInfo();
if (updateInfo.box) {
cloudron.update(updateInfo.box, NOOP_CALLBACK);
} else if (updateInfo.apps) {
apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
}
},
start: true,
timeZone: gBoxUpdateCheckerJob.cronTime.timeZone // hack
src/locker.js
+12 -1
View File
@@ -5,7 +5,10 @@ exports = module.exports = {
unlockForBoxUpdate: unlockForBoxUpdate,
lockForFullBackup: lockForFullBackup,
unlockForFullBackup: unlockForFullBackup
unlockForFullBackup: unlockForFullBackup,
lockForAppTask: lockForAppTask,
unlockForAppTask: unlockForAppTask
};
function lockForBoxUpdate() {
@@ -24,3 +27,11 @@ function unlockForFullBackup() {
return null;
}
function lockForAppTask() {
return null;
}
function unlockForAppTask() {
return null;
}
src/routes/apps.js
+3 -21
View File
@@ -24,8 +24,6 @@ exports = module.exports = {
var apps = require('../apps.js'),
AppsError = apps.AppsError,
assert = require('assert'),
backups = require('../backups.js'),
BackupsError = backups.BackupsError,
debug = require('debug')('box:routes/apps'),
fs = require('fs'),
HttpError = require('connect-lastmile').HttpError,
@@ -177,34 +175,18 @@ function restoreApp(req, res, next) {
function backupApp(req, res, next) {
assert.strictEqual(typeof req.params.id, 'string');
debug('Restore app id:%s', req.params.id);
debug('Backup app id:%s', req.params.id);
backups.scheduleAppBackup(req.params.id, function (error) {
if (error && error.reason === BackupsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
if (error && error.reason === BackupsError.BAD_STATE) return next(new HttpError(409, error.message));
if (error && error.reason === BackupsError.EXTERNAL_ERROR) return next(new HttpError(503, error));
if (error) return next(new HttpError(500, error));
next(new HttpSuccess(202, { }));
});
}
function restoreApp(req, res, next) {
assert.strictEqual(typeof req.params.id, 'string');
debug('Restore app id:%s', req.params.id);
apps.restore(req.params.id, function (error) {
apps.backup(req.params.id, function (error) {
if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error));
if (error) return next(new HttpError(500, error));
next(new HttpSuccess(202, { }));
});
}
/*
* Uninstalls an app
* @bodyparam {string} id The id of the app to be uninstalled
src/routes/backups.js
+2
View File
@@ -16,12 +16,14 @@ var backups = require('../backups.js'),
function get(req, res, next) {
backups.getAllPaged(1, 5, function (error, result) {
if (error) return next(new HttpError(500, error));
next(new HttpSuccess(200, { backups: result }));
});
}
function create(req, res, next) {
// don't want for backup to complete since this can take long
// progress can be checked up ny polling the progress api call
cloudron.backup(function (error) {
if (error) debug('Could not backup', error);
});
src/routes/cloudron.js
+4 -2
View File
@@ -129,8 +129,10 @@ function getConfig(req, res, next) {
}
function update(req, res, next) {
cloudron.update(function (error) {
if (error && error.reason === CloudronError.NO_UPDATE_AVAILABLE) return next(new HttpError(422, 'No update available'));
var boxUpdateInfo = updater.getUpdateInfo().box;
if (!boxUpdateInfo) return next(new HttpError(422, 'No update available'));
cloudron.update(boxUpdateInfo, function (error) {
if (error && error.reason === CloudronError.BAD_STATE) return next(new HttpError(409, error.message));
if (error) return next(new HttpError(500, error));
src/taskmanager.js
+5 -1
View File
@@ -11,6 +11,7 @@ var appdb = require('./appdb.js'),
assert = require('assert'),
child_process = require('child_process'),
debug = require('debug')('box:taskmanager'),
locker = require('./locker.js'),
_ = require('underscore');
var gActiveTasks = { };
@@ -53,7 +54,9 @@ function startAppTask(appId) {
assert.strictEqual(typeof appId, 'string');
assert(!(appId in gActiveTasks));
if (Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
var lockError = locker.lockForAppTask(); // ## FIXME: need to poll when the lock becomes free
if (lockError || Object.keys(gActiveTasks).length >= TASK_CONCURRENCY) {
debug('Reached concurrency limit, queueing task for %s', appId);
gPendingTasks.push(appId);
return;
@@ -66,6 +69,7 @@ function startAppTask(appId) {
appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code }, NOOP_CALLBACK);
}
delete gActiveTasks[appId];
locker.unlockForAppTask();
if (gPendingTasks.length !== 0) startAppTask(gPendingTasks.shift()); // start another pending task
});
}
src/updater.js
+1 -14
View File
@@ -6,21 +6,18 @@ exports = module.exports = {
checkAppUpdates: checkAppUpdates,
checkBoxUpdate: checkBoxUpdates,
getUpdateInfo: getUpdateInfo,
autoupdate: autoupdate
getUpdateInfo: getUpdateInfo
};
var apps = require('./apps.js'),
assert = require('assert'),
async = require('async'),
cloudron = require('./cloudron.js'),
config = require('../config.js'),
debug = require('debug')('box:updater'),
fs = require('fs'),
mailer = require('./mailer.js'),
path = require('path'),
paths = require('./paths.js'),
progress = require('./progress.js'),
safe = require('safetydance'),
semver = require('semver'),
superagent = require('superagent'),
@@ -162,13 +159,3 @@ function checkBoxUpdates() {
});
}
function autoupdate() {
// FIXME: box update and app update must not be concurrent. also, there is no way to track completion of updates
// and this we need to one or the other.
if (gBoxUpdateInfo !== null) {
cloudron.update(NOOP_CALLBACK);
} else {
apps.autoupdateApps(gAppUpdateInfo, NOOP_CALLBACK);
}
}