diff --git a/src/backups.js b/src/backups.js index 97443d7cd..4ad1473ab 100644 --- a/src/backups.js +++ b/src/backups.js @@ -879,11 +879,11 @@ function backupApp(app, progressCallback, callback) { backupAppWithTimestamp(app, timestamp, progressCallback, callback); } -// this function expects you to have a lock -function backupBoxAndApps(auditSource, callback) { +// this function expects you to have a lock. Unlike other progressCallback this also has a progress field +function backupBoxAndApps(auditSource, progressCallback, callback) { assert.strictEqual(typeof auditSource, 'object'); - - callback = callback || NOOP_CALLBACK; + assert.strictEqual(typeof progressCallback, 'function'); + assert.strictEqual(typeof callback, 'function'); var timestamp = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); @@ -892,42 +892,37 @@ function backupBoxAndApps(auditSource, callback) { apps.getAll(function (error, allApps) { if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error)); - var processed = 1; - var step = 100/(allApps.length+2); + let percent = 1; + let step = 100/(allApps.length+2); async.mapSeries(allApps, function iterator(app, iteratorCallback) { - tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Backing up ${app.fqdn}` }, NOOP_CALLBACK); - - ++processed; + progressCallback({ percent: percent, message: `Backing up ${app.fqdn}` }); + percent += step; if (!app.enableBackup) { - tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Skipped backup ${app.fqdn}` }, NOOP_CALLBACK); + debug(`Skipped backup ${app.fqdn}`); return iteratorCallback(null, null); // nothing to backup } - backupAppWithTimestamp(app, timestamp, (progress) => tasks.setProgress(tasks.TASK_BACKUP, { message: progress.message }, NOOP_CALLBACK), function (error, backupId) { + backupAppWithTimestamp(app, timestamp, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) { if (error && error.reason !== BackupsError.BAD_STATE) { debugApp(app, 'Unable to backup', error); return iteratorCallback(error); } - tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: `Backed up ${app.fqdn}` }, NOOP_CALLBACK); + debugApp(app, 'Backed up'); iteratorCallback(null, backupId || null); // clear backupId if is in BAD_STATE and never backed up }); }, function appsBackedUp(error, backupIds) { - if (error) { - tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error.message }, NOOP_CALLBACK); - return callback(error); - } + if (error) return callback(error); backupIds = backupIds.filter(function (id) { return id !== null; }); // remove apps in bad state that were never backed up - tasks.setProgress(tasks.TASK_BACKUP, { percent: step * processed, message: 'Backing up system data' }, NOOP_CALLBACK); - - backupBoxWithAppBackupIds(backupIds, timestamp, (progress) => tasks.setProgress(tasks.TASK_BACKUP, { message: progress.message }, NOOP_CALLBACK), function (error, backupId) { - tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK); + progressCallback({ percent: percent, message: 'Backing up system data' }); + percent += step; + backupBoxWithAppBackupIds(backupIds, timestamp, (progress) => progressCallback({ percent: percent, message: progress.message }), function (error, backupId) { eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, backupId: backupId, timestamp: timestamp }); callback(error, backupId); @@ -943,7 +938,7 @@ function startBackupTask(auditSource, callback) { let error = locker.lock(locker.OP_FULL_BACKUP); if (error) return callback(new BackupsError(BackupsError.BAD_STATE, error.message)); - tasks.setProgress(tasks.TASK_BACKUP, { percent: 0, message: 'Starting' }, NOOP_CALLBACK); // ensure tools can 'wait' on progress + tasks.setProgress(tasks.TASK_BACKUP, { percent: 0, message: 'Starting' }, NOOP_CALLBACK); let fd = safe.fs.openSync(paths.BACKUP_LOG_FILE, 'a'); // will autoclose if (!fd) { @@ -970,9 +965,10 @@ function startBackupTask(auditSource, callback) { gBackupTask = null; + tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK); + locker.unlock(locker.OP_FULL_BACKUP); - tasks.setProgress(tasks.TASK_BACKUP, { percent: 100, result: error ? error.message : '' }, NOOP_CALLBACK); if (error) mailer.backupFailed(error); debug('startBackupTask: backup done'); diff --git a/src/caas.js b/src/caas.js index 5c335beb9..781c17211 100644 --- a/src/caas.js +++ b/src/caas.js @@ -133,7 +133,7 @@ function doMigrate(options, caasConfig, callback) { tasks.setProgress(tasks.TASK_MIGRATE, { percent: 10, result: 'Backing up for migration' }, NOOP_CALLBACK); // initiate the migration in the background - backups.backupBoxAndApps({ userId: null, username: 'migrator' }, function (error) { + backups.backupBoxAndApps({ userId: null, username: 'migrator' }, (progress) => tasks.setProgress(tasks.TASK_MIGRATE, { percent: 10+progress.percent*30/100, message: progress.message }, NOOP_CALLBACK), function (error) { if (error) return unlock(error); debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region); diff --git a/src/tasks/backuptask.js b/src/tasks/backuptask.js index 9532838ce..67afdea8f 100755 --- a/src/tasks/backuptask.js +++ b/src/tasks/backuptask.js @@ -10,7 +10,8 @@ require('supererror')({ splatchError: true }); var assert = require('assert'), backups = require('../backups.js'), database = require('../database.js'), - debug = require('debug')('box:backuptask'); + debug = require('debug')('box:backuptask'), + tasks = require('../tasks.js'); function initialize(callback) { assert.strictEqual(typeof callback, 'function'); @@ -19,6 +20,7 @@ function initialize(callback) { } // Main process starts here +const NOOP_CALLBACK = function (error) { if (error) debug(error); }; const auditSource = JSON.parse(process.argv[2]); debug('Staring complete backup'); @@ -30,7 +32,7 @@ process.on('SIGTERM', function () { initialize(function (error) { if (error) throw error; - backups.backupBoxAndApps(auditSource, function (error) { + backups.backupBoxAndApps(auditSource, (progress) => tasks.setProgress(tasks.TASK_BACKUP, progress, NOOP_CALLBACK), function (error) { if (error) debug('backup failed.', error); process.send({ result: error ? error.message : '' }); diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 96368dac6..f05d1f5d7 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -16,58 +16,12 @@ var async = require('async'), fs = require('fs'), os = require('os'), mkdirp = require('mkdirp'), - readdirp = require('readdirp'), path = require('path'), rimraf = require('rimraf'), settings = require('../settings.js'), SettingsError = require('../settings.js').SettingsError, tasks = require('../tasks.js'); -function compareDirectories(one, two, callback) { - readdirp({ root: one }, function (error, treeOne) { - if (error) return callback(error); - - readdirp({ root: two }, function (error, treeTwo) { - if (error) return callback(error); - - var mismatch = []; - - function compareDirs(a, b) { - a.forEach(function (tmpA) { - var found = b.find(function (tmpB) { - return tmpA.path === tmpB.path; - }); - - if (!found) mismatch.push(tmpA); - }); - } - - function compareFiles(a, b) { - a.forEach(function (tmpA) { - var found = b.find(function (tmpB) { - // TODO check file or symbolic link - return tmpA.path === tmpB.path && tmpA.mode === tmpB.mode; - }); - - if (!found) mismatch.push(tmpA); - }); - } - - compareDirs(treeOne.directories, treeTwo.directories); - compareDirs(treeTwo.directories, treeOne.directories); - compareFiles(treeOne.files, treeTwo.files); - compareFiles(treeTwo.files, treeOne.files); - - if (mismatch.length) { - console.error('Files not found in both: %j', mismatch); - return callback(new Error('file mismatch')); - } - - callback(null); - }); - }); -} - function createBackup(callback) { backups.startBackupTask({ username: 'test' }, function (error) { // this call does not wait for the backup! if (error) return callback(error); diff --git a/src/updater.js b/src/updater.js index 5ab8e730c..73fdd6e7d 100644 --- a/src/updater.js +++ b/src/updater.js @@ -10,7 +10,6 @@ var assert = require('assert'), async = require('async'), child_process = require('child_process'), backups = require('./backups.js'), - caas = require('./caas.js'), config = require('./config.js'), crypto = require('crypto'), debug = require('debug')('box:updater'), @@ -168,7 +167,7 @@ function doUpdate(boxUpdateInfo, callback) { tasks.setProgress(tasks.TASK_UPDATE, { percent: 10, message: 'Backing up' }, NOOP_CALLBACK); - backups.backupBoxAndApps({ userId: null, username: 'updater' }, function (error) { + backups.backupBoxAndApps({ userId: null, username: 'updater' }, (progress) => tasks.setProgress(tasks.TASK_MIGRATE, { percent: 10+progress.percent*70/100, message: progress.message }, NOOP_CALLBACK), function (error) { if (error) return updateError(error); debug('updating box %s', boxUpdateInfo.sourceTarballUrl);