jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Set the backup task progress in the task only

Browse Source
This commit is contained in:
Girish Ramakrishnan
2018-11-27 11:42:44 -08:00
parent e56b87766b
commit b2465dd2ee
5 changed files with 24 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backups.js
+18 -22
View File
@@ -879,11 +879,11 @@ function backupApp(app, progressCallback, callback) {
backupAppWithTimestamp(app, timestamp, progressCallback, callback);
}
// this function expects you to have a lock
function backupBoxAndApps(auditSource, callback) {
// this function expects you to have a lock. Unlike other progressCallback this also has a progress field
function backupBoxAndApps(auditSource, progressCallback, callback) {
assert.strictEqual(typeof auditSource, 'object');
callback = callback || NOOP_CALLBACK;
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
var timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
@@ -892,42 +892,37 @@ function backupBoxAndApps(auditSource, callback) {
apps.getAll(function (error, allApps) {
if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
var processed = 1;
var step = 100/(allApps.length+2);
let percent = 1;
let step = 100/(allApps.length+2);
async.mapSeries(allApps, function iterator(app, iteratorCallback) {
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Backing up ${app.fqdn}` }, NOOP_CALLBACK);
++processed;
progressCallback({ percent: percent, message: `Backing up ${app.fqdn}` });
percent += step;
if (!app.enableBackup) {
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Skipped backup ${app.fqdn}` }, NOOP_CALLBACK);
debug(`Skipped backup ${app.fqdn}`);
return iteratorCallback(null, null); // nothing to backup
}
backupAppWithTimestamp(app, timestamp, (progress) => tasks.setProgress(tasks.TASK_BACKUP, { message: progress.message }, NOOP_CALLBACK), function (error, backupId) {
backupAppWithTimestamp(app, timestamp, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) {
if (error && error.reason !== BackupsError.BAD_STATE) {
debugApp(app, 'Unable to backup', error);
return iteratorCallback(error);
}
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Backed up ${app.fqdn}` }, NOOP_CALLBACK);
debugApp(app, 'Backed up');
iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up
});
}, function appsBackedUp(error, backupIds) {
if (error) {
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error.message }, NOOP_CALLBACK);
return callback(error);
}
if (error) return callback(error);
backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up
tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: 'Backing up system data' }, NOOP_CALLBACK);
backupBoxWithAppBackupIds(backupIds, timestamp, (progress) => tasks.setProgress(tasks.TASK_BACKUP, { message: progress.message }, NOOP_CALLBACK), function (error, backupId) {
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK);
progressCallback({ percent: percent, message: 'Backing up system data' });
percent += step;
backupBoxWithAppBackupIds(backupIds, timestamp, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) {
eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, backupId: backupId, timestamp: timestamp });
callback(error, backupId);
@@ -943,7 +938,7 @@ function startBackupTask(auditSource, callback) {
let error = locker.lock(locker.OP_FULL_BACKUP);
if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message));
tasks.setProgress(tasks.TASK_BACKUP, { percent: 0, message: 'Starting' }, NOOP_CALLBACK); // ensure tools can 'wait' on progress
tasks.setProgress(tasks.TASK_BACKUP, { percent: 0, message: 'Starting' }, NOOP_CALLBACK);
let fd = safe.fs.openSync(paths.BACKUP_LOG_FILE, 'a'); // will autoclose
if (!fd) {
@@ -970,9 +965,10 @@ function startBackupTask(auditSource, callback) {
gBackupTask = null;
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK);
locker.unlock(locker.OP_FULL_BACKUP);
tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK);
if (error) mailer.backupFailed(error);
debug('startBackupTask: backup done');
src/caas.js
+1 -1
View File
@@ -133,7 +133,7 @@ function doMigrate(options, caasConfig, callback) {
tasks.setProgress(tasks.TASK_MIGRATE, { percent: 10, result: 'Backing up for migration' }, NOOP_CALLBACK);
// initiate the migration in the background
backups.backupBoxAndApps({ userId: null, username: 'migrator' }, function (error) {
backups.backupBoxAndApps({ userId: null, username: 'migrator' }, (progress) => tasks.setProgress(tasks.TASK_MIGRATE, { percent: 10+progress.percent*30/100, message: progress.message }, NOOP_CALLBACK), function (error) {
if (error) return unlock(error);
debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region);
src/tasks/backuptask.js
+4 -2
View File
@@ -10,7 +10,8 @@ require('supererror')({ splatchError: true });
var assert = require('assert'),
backups = require('../backups.js'),
database = require('../database.js'),
debug = require('debug')('box:backuptask');
debug = require('debug')('box:backuptask'),
tasks = require('../tasks.js');
function initialize(callback) {
assert.strictEqual(typeof callback, 'function');
@@ -19,6 +20,7 @@ function initialize(callback) {
}
// Main process starts here
const NOOP_CALLBACK = function (error) { if (error) debug(error); };
const auditSource = JSON.parse(process.argv[2]);
debug('Staring complete backup');
@@ -30,7 +32,7 @@ process.on('SIGTERM', function () {
initialize(function (error) {
if (error) throw error;
backups.backupBoxAndApps(auditSource, function (error) {
backups.backupBoxAndApps(auditSource, (progress) => tasks.setProgress(tasks.TASK_BACKUP, progress, NOOP_CALLBACK), function (error) {
if (error) debug('backup failed.', error);
process.send({ result: error ? error.message : '' });
src/test/backups-test.js
-46
View File
@@ -16,58 +16,12 @@ var async = require('async'),
fs = require('fs'),
os = require('os'),
mkdirp = require('mkdirp'),
readdirp = require('readdirp'),
path = require('path'),
rimraf = require('rimraf'),
settings = require('../settings.js'),
SettingsError = require('../settings.js').SettingsError,
tasks = require('../tasks.js');
function compareDirectories(one, two, callback) {
readdirp({ root: one }, function (error, treeOne) {
if (error) return callback(error);
readdirp({ root: two }, function (error, treeTwo) {
if (error) return callback(error);
var mismatch = [];
function compareDirs(a, b) {
a.forEach(function (tmpA) {
var found = b.find(function (tmpB) {
return tmpA.path === tmpB.path;
});
if (!found) mismatch.push(tmpA);
});
}
function compareFiles(a, b) {
a.forEach(function (tmpA) {
var found = b.find(function (tmpB) {
// TODO check file or symbolic link
return tmpA.path === tmpB.path && tmpA.mode === tmpB.mode;
});
if (!found) mismatch.push(tmpA);
});
}
compareDirs(treeOne.directories, treeTwo.directories);
compareDirs(treeTwo.directories, treeOne.directories);
compareFiles(treeOne.files, treeTwo.files);
compareFiles(treeTwo.files, treeOne.files);
if (mismatch.length) {
console.error('Files not found in both: %j', mismatch);
return callback(new Error('file mismatch'));
}
callback(null);
});
});
}
function createBackup(callback) {
backups.startBackupTask({ username: 'test' }, function (error) { // this call does not wait for the backup!
if (error) return callback(error);
src/updater.js
+1 -2
View File
@@ -10,7 +10,6 @@ var assert = require('assert'),
async = require('async'),
child_process = require('child_process'),
backups = require('./backups.js'),
caas = require('./caas.js'),
config = require('./config.js'),
crypto = require('crypto'),
debug = require('debug')('box:updater'),
@@ -168,7 +167,7 @@ function doUpdate(boxUpdateInfo, callback) {
tasks.setProgress(tasks.TASK_UPDATE, { percent: 10, message: 'Backing up' }, NOOP_CALLBACK);
backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) {
backups.backupBoxAndApps({ userId: null, username: 'updater' }, (progress) => tasks.setProgress(tasks.TASK_MIGRATE, { percent: 10+progress.percent*70/100, message: progress.message }, NOOP_CALLBACK), function (error) {
if (error) return updateError(error);
debug('updating box %s', boxUpdateInfo.sourceTarballUrl);