jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

rework backup code

Browse Source 
move all the backup code into backups.js
This commit is contained in:
Girish Ramakrishnan
2016-04-10 19:17:44 -07:00
parent af9d489395
commit 322e9faee7
6 changed files with 301 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

294
src/backups.js
View File

@@ -12,17 +12,61 @@ exports = module.exports = {
copyLastBackup: copyLastBackup,
getBackupCredentials: getBackupCredentials
ensureBackup: ensureBackup,
backup: backup,
backupApp: backupApp,
restoreApp: restoreApp,
getBackupCredentials: getBackupCredentials,
backupBoxAndApps: backupBoxAndApps
};
var assert = require('assert'),
var addons = require('./addons.js'),
appdb = require('./appdb.js'),
apps = require('./apps.js'),
async = require('async'),
assert = require('assert'),
backupdb = require('./backupdb.js'),
caas = require('./storage/caas.js'),
config = require('./config.js'),
DatabaseError = require('./databaseerror.js'),
debug = require('debug')('box:backups'),
locker = require('./locker.js'),
path = require('path'),
paths = require('./paths.js'),
progress = require('./progress.js'),
s3 = require('./storage/s3.js'),
safe = require('safetydance'),
shell = require('./shell.js'),
settings = require('./settings.js'),
util = require('util');
util = require('util'),
webhooks = require('./webhooks.js');
var BACKUP_BOX_CMD = path.join(__dirname, 'scripts/backupbox.sh'),
BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh'),
BACKUP_APP_CMD = path.join(__dirname, 'scripts/backupapp.sh'),
RESTORE_APP_CMD = path.join(__dirname, 'scripts/restoreapp.sh'),
BACKUP_SWAP_CMD = path.join(__dirname, 'scripts/backupswap.sh');
var NOOP_CALLBACK = function (error) { if (error) debug(error); };
function debugApp(app, args) {
assert(!app || typeof app === 'object');
var prefix = app ? app.location : '(no app)';
debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
}
function ignoreError(func) {
return function (callback) {
func(function (error) {
if (error) console.error('Ignored error:', error);
callback();
});
};
}
function BackupsError(reason, errorOrMessage) {
assert.strictEqual(typeof reason, 'string');
@@ -45,6 +89,7 @@ function BackupsError(reason, errorOrMessage) {
util.inherits(BackupsError, Error);
BackupsError.EXTERNAL_ERROR = 'external error';
BackupsError.INTERNAL_ERROR = 'internal error';
BackupsError.BAD_STATE = 'bad state';
BackupsError.MISSING_CREDENTIALS = 'missing credentials';
// choose which storage backend we use for test purpose we use s3
@@ -203,3 +248,246 @@ function copyLastBackup(app, callback) {
});
});
}
function backupBoxWithAppBackupIds(appBackupIds, callback) {
assert(util.isArray(appBackupIds));
getBackupUrl(appBackupIds, function (error, result) {
if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
debug('backupBoxWithAppBackupIds: %j', result);
async.series([
ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
shell.sudo.bind(null, 'backupBox', [ BACKUP_BOX_CMD, result.s3Url, result.accessKeyId, result.secretAccessKey, result.sessionToken, result.region, result.backupKey ]),
ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
], function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
debug('backupBoxWithAppBackupIds: success');
webhooks.backupDone(result.id, null /* app */, appBackupIds, function (error) {
if (error) return callback(error);
callback(null, result.id);
});
});
});
}
// this function expects you to have a lock
// function backupBox(callback) {
// apps.getAll(function (error, allApps) {
// if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
//
// var appBackupIds = allApps.map(function (app) { return app.lastBackupId; });
// appBackupIds = appBackupIds.filter(function (id) { return id !== null; }); // remove apps that were never backed up
//
// backupBoxWithAppBackupIds(appBackupIds, callback);
// });
// }
// this function expects you to have a lock
function backupBoxAndApps(callback) {
callback = callback || NOOP_CALLBACK;
apps.getAll(function (error, allApps) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
var processed = 0;
var step = 100/(allApps.length+1);
progress.set(progress.BACKUP, processed, '');
async.mapSeries(allApps, function iterator(app, iteratorCallback) {
++processed;
backupApp(app, app.manifest.addons, function (error, backupId) {
if (error && error.reason !== BackupsError.BAD_STATE) {
debugApp(app, 'Unable to backup', error);
return iteratorCallback(error);
}
progress.set(progress.BACKUP, step * processed, 'Backed up app at ' + app.location);
iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
});
}, function appsBackedUp(error, backupIds) {
if (error) {
progress.set(progress.BACKUP, 100, error.message);
return callback(error);
}
backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
backupBoxWithAppBackupIds(backupIds, function (error, filename) {
progress.set(progress.BACKUP, 100, error ? error.message : '');
callback(error, filename);
});
});
});
}
function backup(callback) {
assert.strictEqual(typeof callback, 'function');
var error = locker.lock(locker.OP_FULL_BACKUP);
if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
// ensure tools can 'wait' on progress
progress.set(progress.BACKUP, 0, 'Starting');
// start the backup operation in the background
backupBoxAndApps(function (error) {
if (error) console.error('backup failed.', error);
locker.unlock(locker.OP_FULL_BACKUP);
});
callback(null);
}
function ensureBackup(callback) {
callback = callback || NOOP_CALLBACK;
getPaged(1, 1, function (error, backups) {
if (error) {
debug('Unable to list backups', error);
return callback(error); // no point trying to backup if appstore is down
}
if (backups.length !== 0 && (new Date() - new Date(backups[0].creationTime) < 23 * 60 * 60 * 1000)) { // ~1 day ago
debug('Previous backup was %j, no need to backup now', backups[0]);
return callback(null);
}
backup(callback);
});
}
function canBackupApp(app) {
// only backup apps that are installed or pending configure or called from apptask. Rest of them are in some
// state not good for consistent backup (i.e addons may not have been setup completely)
return (app.installationState === appdb.ISTATE_INSTALLED && app.health === appdb.HEALTH_HEALTHY) ||
app.installationState === appdb.ISTATE_PENDING_CONFIGURE ||
app.installationState === appdb.ISTATE_PENDING_BACKUP || // called from apptask
app.installationState === appdb.ISTATE_PENDING_UPDATE; // called from apptask
}
// set the 'creation' date of lastBackup so that the backup persists across time based archival rules
// s3 does not allow changing creation time, so copying the last backup is easy way out for now
function reuseOldBackup(app, callback) {
assert.strictEqual(typeof app.lastBackupId, 'string');
assert.strictEqual(typeof callback, 'function');
copyLastBackup(app, function (error, newBackupId) {
if (error) return callback(error);
debugApp(app, 'reuseOldBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
callback(null, newBackupId);
});
}
function createNewBackup(app, addonsToBackup, callback) {
assert.strictEqual(typeof app, 'object');
assert(!addonsToBackup || typeof addonsToBackup, 'object');
assert.strictEqual(typeof callback, 'function');
getAppBackupUrl(app, function (error, result) {
if (error) return callback(error);
debugApp(app, 'backupApp: backup url:%s backup config url:%s', result.url, result.configUrl);
async.series([
ignoreError(shell.sudo.bind(null, 'mountSwap', [ BACKUP_SWAP_CMD, '--on' ])),
addons.backupAddons.bind(null, app, addonsToBackup),
shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD, result.s3ConfigUrl, result.s3DataUrl, result.accessKeyId, result.secretAccessKey, result.sessionToken, result.region, result.backupKey ]),
ignoreError(shell.sudo.bind(null, 'unmountSwap', [ BACKUP_SWAP_CMD, '--off' ])),
], function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
callback(null, result.id);
});
});
}
function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
assert.strictEqual(typeof appId, 'string');
assert.strictEqual(typeof lastBackupId, 'string');
assert.strictEqual(typeof lastBackupConfig, 'object');
assert.strictEqual(typeof callback, 'function');
appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
return callback(null);
});
}
function backupApp(app, addonsToBackup, callback) {
assert.strictEqual(typeof app, 'object');
assert(!addonsToBackup || typeof addonsToBackup, 'object');
assert.strictEqual(typeof callback, 'function');
var appConfig = null, backupFunction;
if (!canBackupApp(app)) {
if (!app.lastBackupId) {
debugApp(app, 'backupApp: cannot backup app');
return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
}
appConfig = app.lastBackupConfig;
backupFunction = reuseOldBackup.bind(null, app);
} else {
appConfig = {
manifest: app.manifest,
location: app.location,
portBindings: app.portBindings,
accessRestriction: app.accessRestriction,
memoryLimit: app.memoryLimit
};
backupFunction = createNewBackup.bind(null, app, addonsToBackup);
if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
return callback(safe.error);
}
}
backupFunction(function (error, backupId) {
if (error) return callback(error);
debugApp(app, 'backupApp: successful id:%s', backupId);
setRestorePoint(app.id, backupId, appConfig, function (error) {
if (error) return callback(error);
return callback(null, backupId);
});
});
}
function restoreApp(app, addonsToRestore, backupId, callback) {
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof addonsToRestore, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof callback, 'function');
assert(app.lastBackupId);
getRestoreUrl(backupId, function (error, result) {
if (error) return callback(error);
debugApp(app, 'restoreApp: restoreUrl:%s', result.url);
shell.sudo('restoreApp', [ RESTORE_APP_CMD, app.id, result.url, result.backupKey, result.sessionToken ], function (error) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
addons.restoreAddons(app, addonsToRestore, callback);
});
});
}