diff --git a/src/backupcleaner.js b/src/backupcleaner.js index 25eff8756..4e2885b94 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -22,14 +22,14 @@ const apps = require('./apps.js'), util = require('util'), _ = require('underscore'); -function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) { - assert(Array.isArray(backups)); +function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) { + assert(Array.isArray(allBackups)); assert.strictEqual(typeof policy, 'object'); assert(Array.isArray(referencedBackupIds)); const now = new Date(); - for (const backup of backups) { + for (const backup of allBackups) { if (backup.state === backups.BACKUP_STATE_ERROR) { backup.discardReason = 'error'; } else if (backup.state === backups.BACKUP_STATE_CREATING) { @@ -58,7 +58,7 @@ function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) { if (!n) continue; // disabled rule let lastPeriod = null, keptSoFar = 0; - for (const backup of backups) { + for (const backup of allBackups) { if (backup.discardReason) continue; // already discarded for some reason if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason const period = moment(backup.creationTime).format(KEEP_FORMATS[format]); @@ -71,11 +71,11 @@ function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) { } if (policy.keepLatest) { - let latestNormalBackup = backups.find(b => b.state === backups.BACKUP_STATE_NORMAL); + let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL); if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest'; } - for (const backup of backups) { + for (const backup of allBackups) { debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`); } } @@ -279,21 +279,20 @@ function run(progressCallback, callback) { progressCallback({ percent: 40, message: 'Cleaning app backups' }); - cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, function (error, removedAppBackupIds) { + cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, async function (error, removedAppBackupIds) { if (error) return callback(error); progressCallback({ percent: 70, message: 'Cleaning missing backups' }); - cleanupMissingBackups(backupConfig, progressCallback, function (error, missingBackupIds) { + const [cleanupMissingBackupsError, missingBackupIds] = await cleanupMissingBackups(backupConfig, progressCallback); + if (cleanupMissingBackupsError) return callback(cleanupMissingBackupsError); + + progressCallback({ percent: 90, message: 'Cleaning snapshots' }); + + cleanupSnapshots(backupConfig, function (error) { if (error) return callback(error); - progressCallback({ percent: 90, message: 'Cleaning snapshots' }); - - cleanupSnapshots(backupConfig, function (error) { - if (error) return callback(error); - - callback(null, { removedBoxBackupIds, removedAppBackupIds, missingBackupIds }); - }); + callback(null, { removedBoxBackupIds, removedAppBackupIds, missingBackupIds }); }); }); }); diff --git a/src/backuptask.js b/src/backuptask.js index 3093b2749..2887076bc 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -376,10 +376,9 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) { } // this is not part of 'snapshotting' because we need root access to traverse -function saveFsMetadata(dataLayout, metadataFile, callback) { +async function saveFsMetadata(dataLayout, metadataFile) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof metadataFile, 'string'); - assert.strictEqual(typeof callback, 'function'); // contains paths prefixed with './' let metadata = { @@ -391,24 +390,22 @@ function saveFsMetadata(dataLayout, metadataFile, callback) { // we assume small number of files. spawnSync will raise a ENOBUFS error after maxBuffer for (let lp of dataLayout.localPaths()) { const emptyDirs = safe.child_process.execSync(`find ${lp} -type d -empty`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); - if (emptyDirs === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`)); + if (emptyDirs === null) throw new BoxError(BoxError.FS_ERROR, `Error finding empty dirs: ${safe.error.message}`); if (emptyDirs.length) metadata.emptyDirs = metadata.emptyDirs.concat(emptyDirs.trim().split('\n').map((ed) => dataLayout.toRemotePath(ed))); const execFiles = safe.child_process.execSync(`find ${lp} -type f -executable`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); - if (execFiles === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`)); + if (execFiles === null) throw new BoxError(BoxError.FS_ERROR, `Error finding executables: ${safe.error.message}`); if (execFiles.length) metadata.execFiles = metadata.execFiles.concat(execFiles.trim().split('\n').map((ef) => dataLayout.toRemotePath(ef))); const symlinks = safe.child_process.execSync(`find ${lp} -type l`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 30 }); - if (symlinks === null) return callback(new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`)); + if (symlinks === null) throw new BoxError(BoxError.FS_ERROR, `Error finding symlinks: ${safe.error.message}`); if (symlinks.length) metadata.symlinks = metadata.symlinks.concat(symlinks.trim().split('\n').map((sl) => { const target = safe.fs.readlinkSync(sl); return { path: dataLayout.toRemotePath(sl), target }; })); } - if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) return callback(new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`)); - - callback(); + if (!safe.fs.writeFileSync(metadataFile, JSON.stringify(metadata, null, 4))) throw new BoxError(BoxError.FS_ERROR, `Error writing fs metadata: ${safe.error.message}`); } // this function is called via backupupload (since it needs root to traverse app's directory) @@ -448,7 +445,7 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback) }, callback); } else { async.series([ - saveFsMetadata.bind(null, dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`), + saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`), sync.bind(null, backupConfig, backupId, dataLayout, progressCallback) ], callback); } @@ -512,43 +509,35 @@ function tarExtract(inStream, dataLayout, encryption, callback) { callback(null, ps); } -function restoreFsMetadata(dataLayout, metadataFile, callback) { +async function restoreFsMetadata(dataLayout, metadataFile) { assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof metadataFile, 'string'); - assert.strictEqual(typeof callback, 'function'); debug(`Recreating empty directories in ${dataLayout.toString()}`); var metadataJson = safe.fs.readFileSync(metadataFile, 'utf8'); - if (metadataJson === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message)); + if (metadataJson === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error loading fsmetadata.json:' + safe.error.message); var metadata = safe.JSON.parse(metadataJson); - if (metadata === null) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message)); + if (metadata === null) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Error parsing fsmetadata.json:' + safe.error.message); - async.eachSeries(metadata.emptyDirs, function createPath(emptyDir, iteratorDone) { - fs.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true }, iteratorDone); - }, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to create path: ${error.message}`)); + for (const emptyDir of metadata.emptyDirs) { + const [mkdirError] = await safe(fs.promises.mkdir(dataLayout.toLocalPath(emptyDir), { recursive: true })); + if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to create path: ${mkdirError.message}`); + } - async.eachSeries(metadata.execFiles, function createPath(execFile, iteratorDone) { - fs.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8), iteratorDone); - }, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to chmod: ${error.message}`)); + for (const execFile of metadata.execFiles) { + const [chmodError] = await safe(fs.promises.chmod(dataLayout.toLocalPath(execFile), parseInt('0755', 8))); + if (chmodError) throw new BoxError(BoxError.FS_ERROR, `unable to chmod: ${chmodError.message}`); + } - async.eachSeries(metadata.symlinks || [], function createSymlink(symlink, iteratorDone) { - if (!symlink.target) return iteratorDone(); - // the path may not exist if we had a directory full of symlinks - fs.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true }, function (error) { - if (error) return iteratorDone(error); - - fs.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file', iteratorDone); - }); - }, function (error) { - if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `unable to symlink: ${error.message}`)); - - callback(); - }); - }); - }); + for (const symlink of (metadata.symlinks || [])) { + if (!symlink.target) continue; + // the path may not exist if we had a directory full of symlinks + const [mkdirError] = await safe(fs.promises.mkdir(path.dirname(dataLayout.toLocalPath(symlink.path)), { recursive: true })); + if (mkdirError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink (mkdir): ${mkdirError.message}`); + const [symlinkError] = await safe(fs.promises.symlink(symlink.target, dataLayout.toLocalPath(symlink.path), 'file')); + if (symlinkError) throw new BoxError(BoxError.FS_ERROR, `unable to symlink: ${symlinkError.message}`); + } } function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) { @@ -646,10 +635,11 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback, }); }, callback); } else { - downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, function (error) { + downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, async function (error) { if (error) return callback(error); - restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, callback); + [error] = await safe(restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`)); + callback(error); }); } } @@ -843,8 +833,8 @@ function backupBoxWithAppBackupIds(appBackupIds, tag, options, progressCallback, uploadBoxSnapshot(backupConfig, progressCallback, async function (error) { if (error) return callback(error); - const [rotateError] = await safe(rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback)); - callback(rotateError); + const [rotateError, backupId] = await safe(rotateBoxBackup(backupConfig, tag, options, appBackupIds, progressCallback)); + callback(rotateError, backupId); }); }); } @@ -990,8 +980,8 @@ async function backupAppWithTag(app, tag, options, progressCallback, callback) { uploadAppSnapshot(backupConfig, app, progressCallback, async function (error) { if (error) return callback(error); - const [rotateError] = await safe(rotateAppBackup(backupConfig, app, tag, options, progressCallback)); - callback(rotateError); + const [rotateError, backupId] = await safe(rotateAppBackup(backupConfig, app, tag, options, progressCallback)); + callback(rotateError, backupId); }); }); } diff --git a/src/cron.js b/src/cron.js index 68a3be184..5f72b4e3b 100644 --- a/src/cron.js +++ b/src/cron.js @@ -95,7 +95,7 @@ function startJobs(callback) { gJobs.cleanupBackups = new CronJob({ cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN, - onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK), + onTick: backups.startCleanupTask.bind(null, auditSource.CRON), start: true }); diff --git a/src/test/backupcleaner-test.js b/src/test/backupcleaner-test.js new file mode 100644 index 000000000..349649d73 --- /dev/null +++ b/src/test/backupcleaner-test.js @@ -0,0 +1,280 @@ +/* jslint node:true */ +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +'use strict'; + +const backupCleaner = require('../backupcleaner.js'), + backups = require('../backups.js'), + common = require('./common.js'), + delay = require('delay'), + expect = require('expect.js'), + moment = require('moment'), + settings = require('../settings.js'), + settingsdb = require('../settingsdb.js'), + tasks = require('../tasks.js'); + +describe('backup cleaner', function () { + const { setup, cleanup, APP } = common; + + before(setup); + after(cleanup); + + const backupTemplate = { + id: 'someid', + encryptionVersion: 2, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_BOX, + state: backups.BACKUP_STATE_NORMAL, + identifier: 'box', + dependsOn: [ 'dep1' ], + manifest: null, + format: 'tgz', + preserveSecs: 0 + }; + + describe('retention policy', function () { + it('keeps latest', function () { + const backup = Object.assign({}, backupTemplate, { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }); + backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); + expect(backup.keepReason).to.be('latest'); + }); + + it('does not keep latest', function () { + let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; + backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []); + expect(backup.keepReason).to.be(undefined); + }); + + it('always keeps forever policy', function () { + let backup = { creationTime: new Date() }; + backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []); + expect(backup.keepReason).to.be('keepWithinSecs'); + }); + + it('preserveSecs takes precedence', function () { + let backup = { creationTime: new Date(), preserveSecs: 3000 }; + backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); + expect(backup.keepReason).to.be('preserveSecs'); + }); + + it('1 daily', function () { + let b = [ + { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, + { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } + ]; + backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []); + expect(b[0].keepReason).to.be('keepDaily'); + expect(b[1].keepReason).to.be(undefined); + expect(b[2].keepReason).to.be(undefined); + expect(b[3].keepReason).to.be(undefined); + expect(b[3].keepReason).to.be(undefined); + }); + + // if you are debugging this test, it's because of some timezone issue with all the hour substraction! + it('2 daily, 1 weekly', function () { + let b = [ + { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, + { id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() }, + { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, + { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } + ]; + backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); + expect(b[0].keepReason).to.be('keepDaily'); // today + expect(b[1].keepReason).to.be('keepWeekly'); // today + expect(b[2].keepReason).to.be(undefined); + expect(b[3].keepReason).to.be('keepDaily'); // yesterday + expect(b[4].discardReason).to.be('error'); // errored + expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long + expect(b[6].keepReason).to.be(undefined); // outside retention policy + }); + + it('2 daily, 3 monthly, 1 yearly', function () { + let b = [ + { id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() }, + { id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() }, + { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, + { id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() }, + { id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() }, + { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, + { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, + ]; + backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); + expect(b[0].keepReason).to.be('creating'); + expect(b[1].discardReason).to.be('error'); // errored + expect(b[2].keepReason).to.be('keepDaily'); + expect(b[3].keepReason).to.be('keepMonthly'); + expect(b[4].keepReason).to.be('keepYearly'); + expect(b[5].keepReason).to.be('keepDaily'); // yesterday + expect(b[6].discardReason).to.be('creating-too-long'); // errored + expect(b[7].keepReason).to.be('keepMonthly'); + expect(b[8].keepReason).to.be('keepMonthly'); + expect(b[9].keepReason).to.be(undefined); + }); + }); + + describe('task', function () { + const BACKUP_0_BOX = { + id: 'backup-box-0', + identifier: 'box', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_BOX, + state: backups.BACKUP_STATE_NORMAL, + dependsOn: [ 'backup-app-00', 'backup-app-01' ], + manifest: null, + format: 'tgz' + }; + + const BACKUP_0_APP_0 = { // backup of installed app + id: 'backup-app-00', + identifier: APP.id, + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_NORMAL, + dependsOn: [], + manifest: null, + format: 'tgz' + }; + + const BACKUP_0_APP_1 = { // this app is uninstalled + id: 'backup-app-01', + identifier: 'app1', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_NORMAL, + dependsOn: [], + manifest: null, + format: 'tgz' + }; + + const BACKUP_1_BOX = { + id: 'backup-box-1', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_BOX, + state: backups.BACKUP_STATE_NORMAL, + identifier: 'box', + dependsOn: [ 'backup-app-10', 'backup-app-11' ], + manifest: null, + format: 'tgz' + }; + + const BACKUP_1_APP_0 = { + id: 'backup-app-10', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_NORMAL, + identifier: APP.id, + dependsOn: [], + manifest: null, + format: 'tgz' + }; + + const BACKUP_1_APP_1 = { + id: 'backup-app-11', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_NORMAL, + identifier: 'app1', + dependsOn: [], + manifest: null, + format: 'tgz' + }; + + before(function (done) { + settingsdb.set(settings.BACKUP_CONFIG_KEY, JSON.stringify({ + provider: 'filesystem', + password: 'supersecret', + backupFolder: '/tmp/someplace', + retentionPolicy: { keepWithinSecs: 1 }, + format: 'tgz' + }), done); + }); + + async function cleanupBackups() { + const taskId = await backups.startCleanupTask({ username: 'test' }); + + console.log('started task', taskId); + + // eslint-disable-next-line no-constant-condition + while (true) { + await delay(1000); + + const p = await tasks.get(taskId); + + if (p.percent !== 100) continue; + if (p.error) throw new Error(`backup failed: ${p.error.message}`); + + return; + } + } + + it('succeeds without backups', async function () { + await cleanupBackups(); + }); + + it('succeeds with box backups, keeps latest', async function () { + for (const backup of [[ BACKUP_0_BOX, BACKUP_0_APP_0, BACKUP_0_APP_1 ], [ BACKUP_1_BOX, BACKUP_1_APP_0, BACKUP_1_APP_1 ]]) { + await delay(2000); // space out backups + for (const b of backup) { + await backups.add(b.id, b); + } + } + + await cleanupBackups(); + + const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000); + expect(results.length).to.equal(1); + expect(results[0].id).to.equal(BACKUP_1_BOX.id); + + // check that app backups are gone as well. only backup_1 will remain + const result = await backups.get(BACKUP_0_APP_0.id); + expect(result).to.be(null); + }); + + it('does not remove expired backups if only one left', async function () { + await cleanupBackups(); + + const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000); + expect(results[0].id).to.equal(BACKUP_1_BOX.id); + + // check that app backups are also still there. backup_1 is still there + const result = await backups.get(BACKUP_1_APP_0.id); + expect(result.id).to.equal(BACKUP_1_APP_0.id); + }); + + it('succeeds for app backups not referenced by a box backup', async function () { + // add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there + for (const backup of [BACKUP_0_APP_0, BACKUP_0_APP_1]) { + await backups.add(backup.id, backup); + } + + await delay(2000); // wait for expiration + + await cleanupBackups(); + + let result = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000); + expect(result.length).to.equal(3); + result = result.sort((r1, r2) => r1.id.localeCompare(r2.id)); + expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved + expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box + expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box + }); + }); +}); diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 5e0a8db85..0a7d1c6fb 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -6,585 +6,97 @@ 'use strict'; -const async = require('async'), - backupdb = require('../backupdb.js'), - backups = require('../backups.js'), +const backups = require('../backups.js'), BoxError = require('../boxerror.js'), common = require('./common.js'), - DataLayout = require('../datalayout.js'), - delay = require('delay'), expect = require('expect.js'), - fs = require('fs'), - os = require('os'), - moment = require('moment'), - path = require('path'), - rimraf = require('rimraf'), - safe = require('safetydance'), - settingsdb = require('../settingsdb.js'), - settings = require('../settings.js'), - tasks = require('../tasks.js'), - util = require('util'); - -const { createTree, APP } = common; - -function createBackup(callback) { - backups.startBackupTask({ username: 'test' }, function (error, taskId) { // this call does not wait for the backup! - if (error) return callback(error); - - async function waitForBackup() { - const [error, p] = await safe(tasks.get(taskId)); - if (error) return callback(error); - - if (p.percent !== 100) return setTimeout(waitForBackup, 1000); - - if (p.errorMessage) return callback(new Error('backup failed:' + p)); - if (!p.result) return callback(new Error('backup has no result:' + p)); - - backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, 1, 1, function (error, result) { - if (error) return callback(error); - if (result.length !== 1) return callback(new Error('result is not of length 1')); - - // the task progress and the db entry is set in the worker. wait for 2 seconds for backup lock to get released in parent process - setTimeout(() => callback(null, result[0]), 2000); - }); - } - - setTimeout(waitForBackup, 1000); - }); -} - -async function cleanupBackups() { - const taskId = await backups.startCleanupTask({ username: 'test' }); - - // eslint-disable-next-line no-constant-condition - while (true) { - await delay(1000); - - const p = await tasks.get(taskId); - - if (p.percent !== 100) continue; - if (p.errorMessage) throw new Error('backup failed:' + p.errorMessage); - - return; - } -} + safe = require('safetydance'); describe('backups', function () { - before(function (done) { - const BACKUP_DIR = path.join(os.tmpdir(), 'cloudron-backup-test'); + const { setup, cleanup } = common; - async.series([ - common.setup, - fs.mkdir.bind(null, BACKUP_DIR, { recursive: true }), - settingsdb.set.bind(null, settings.BACKUP_CONFIG_KEY, JSON.stringify({ - provider: 'filesystem', - password: 'supersecret', - backupFolder: BACKUP_DIR, - retentionPolicy: { keepWithinSecs: 1 }, - format: 'tgz' - })) - ], done); + before(setup); + after(cleanup); + + const boxBackup = { + id: 'backup-box', + encryptionVersion: 2, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_BOX, + state: backups.BACKUP_STATE_NORMAL, + identifier: 'box', + dependsOn: [ 'dep1' ], + manifest: null, + format: 'tgz', + preserveSecs: 0 + }; + + const appBackup = { + id: 'app_appid_123', + encryptionVersion: null, + packageVersion: '1.0.0', + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_CREATING, + identifier: 'appid', + dependsOn: [ ], + manifest: { foo: 'bar' }, + format: 'tgz', + preserveSecs: 0 + }; + + it('add succeeds', async function () { + await backups.add(boxBackup.id, boxBackup); }); - after(common.cleanup); - - describe('backup', function () { - - it('add succeeds', function (done) { - var backup = { - id: 'backup-box', - encryptionVersion: 2, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, - identifier: 'box', - dependsOn: [ 'dep1' ], - manifest: null, - format: 'tgz' - }; - - backupdb.add(backup.id, backup, function (error) { - expect(error).to.be(null); - done(); - }); - }); - - it('get succeeds', function (done) { - backupdb.get('backup-box', function (error, result) { - expect(error).to.be(null); - expect(result.encryptionVersion).to.be(2); - expect(result.packageVersion).to.be('1.0.0'); - expect(result.type).to.be(backups.BACKUP_TYPE_BOX); - expect(result.state).to.be(backups.BACKUP_STATE_NORMAL); - expect(result.creationTime).to.be.a(Date); - expect(result.dependsOn).to.eql(['dep1']); - expect(result.manifest).to.eql(null); - done(); - }); - }); - - it('get of unknown id fails', function (done) { - backupdb.get('somerandom', function (error, result) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.be(BoxError.NOT_FOUND); - expect(result).to.not.be.ok(); - done(); - }); - }); - - it('getByTypePaged succeeds', function (done) { - backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5, function (error, results) { - expect(error).to.be(null); - expect(results).to.be.an(Array); - expect(results.length).to.be(1); - - expect(results[0].id).to.be('backup-box'); - expect(results[0].encryptionVersion).to.be(2); - expect(results[0].packageVersion).to.be('1.0.0'); - expect(results[0].dependsOn).to.eql(['dep1']); - expect(results[0].manifest).to.eql(null); - - done(); - }); - }); - - it('delete succeeds', function (done) { - backupdb.del('backup-box', function (error, result) { - expect(error).to.be(null); - expect(result).to.not.be.ok(); - - backupdb.get('backup-box', function (error, result) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.equal(BoxError.NOT_FOUND); - expect(result).to.not.be.ok(); - done(); - }); - }); - }); - - it('add app succeeds', function (done) { - var backup = { - id: 'app_appid_123', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_CREATING, - identifier: 'appid', - dependsOn: [ ], - manifest: { foo: 'bar' }, - format: 'tgz' - }; - - backupdb.add(backup.id, backup, function (error) { - expect(error).to.be(null); - done(); - }); - }); - - it('get succeeds', function (done) { - backupdb.get('app_appid_123', function (error, result) { - expect(error).to.be(null); - expect(result.encryptionVersion).to.be(null); - expect(result.packageVersion).to.be('1.0.0'); - expect(result.type).to.be(backups.BACKUP_TYPE_APP); - expect(result.state).to.be(backups.BACKUP_STATE_CREATING); - expect(result.creationTime).to.be.a(Date); - expect(result.dependsOn).to.eql([]); - expect(result.manifest).to.eql({ foo: 'bar' }); - done(); - }); - }); - - it('getByIdentifierPaged succeeds', function (done) { - backupdb.getByIdentifierPaged('appid', 1, 5, function (error, results) { - expect(error).to.be(null); - expect(results).to.be.an(Array); - expect(results.length).to.be(1); - - expect(results[0].id).to.be('app_appid_123'); - expect(results[0].encryptionVersion).to.be(null); - expect(results[0].packageVersion).to.be('1.0.0'); - expect(results[0].dependsOn).to.eql([]); - expect(results[0].manifest).to.eql({ foo: 'bar' }); - - done(); - }); - }); - - it('delete succeeds', function (done) { - backupdb.del('app_appid_123', function (error, result) { - expect(error).to.be(null); - expect(result).to.not.be.ok(); - - backupdb.get('app_appid_123', function (error, result) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.equal(BoxError.NOT_FOUND); - expect(result).to.not.be.ok(); - done(); - }); - }); - }); - + it('fails with duplicating id', async function () { + const [error] = await safe(backups.add(boxBackup.id, boxBackup)); + expect(error.reason).to.be(BoxError.ALREADY_EXISTS); }); - describe('retention policy', function () { - it('keeps latest', function () { - let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); - expect(backup.keepReason).to.be('latest'); - }); - - it('does not keep latest', function () { - let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []); - expect(backup.keepReason).to.be(undefined); - }); - - it('always keeps forever policy', function () { - let backup = { creationTime: new Date() }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []); - expect(backup.keepReason).to.be('keepWithinSecs'); - }); - - it('preserveSecs takes precedence', function () { - let backup = { creationTime: new Date(), preserveSecs: 3000 }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); - expect(backup.keepReason).to.be('preserveSecs'); - }); - - it('1 daily', function () { - let b = [ - { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, - { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, - { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, - { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } - ]; - backups._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []); - expect(b[0].keepReason).to.be('keepDaily'); - expect(b[1].keepReason).to.be(undefined); - expect(b[2].keepReason).to.be(undefined); - expect(b[3].keepReason).to.be(undefined); - expect(b[3].keepReason).to.be(undefined); - }); - - // if you are debugging this test, it's because of some timezone issue with all the hour substraction! - it('2 daily, 1 weekly', function () { - let b = [ - { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, - { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, - { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, - { id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() }, - { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, - { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } - ]; - backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); - expect(b[0].keepReason).to.be('keepDaily'); // today - expect(b[1].keepReason).to.be('keepWeekly'); // today - expect(b[2].keepReason).to.be(undefined); - expect(b[3].keepReason).to.be('keepDaily'); // yesterday - expect(b[4].discardReason).to.be('error'); // errored - expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long - expect(b[6].keepReason).to.be(undefined); // outside retention policy - }); - - it('2 daily, 3 monthly, 1 yearly', function () { - let b = [ - { id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() }, - { id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() }, - { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, - { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, - { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, - { id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, - { id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() }, - { id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() }, - { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, - { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, - ]; - backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); - expect(b[0].keepReason).to.be('creating'); - expect(b[1].discardReason).to.be('error'); // errored - expect(b[2].keepReason).to.be('keepDaily'); - expect(b[3].keepReason).to.be('keepMonthly'); - expect(b[4].keepReason).to.be('keepYearly'); - expect(b[5].keepReason).to.be('keepDaily'); // yesterday - expect(b[6].discardReason).to.be('creating-too-long'); // errored - expect(b[7].keepReason).to.be('keepMonthly'); - expect(b[8].keepReason).to.be('keepMonthly'); - expect(b[9].keepReason).to.be(undefined); - }); - + it('get succeeds', async function () { + const result = await backups.get(boxBackup.id); + delete result.creationTime; + expect(result).to.eql(boxBackup); }); - describe('cleanup', function () { - var BACKUP_0 = { - id: 'backup-box-0', - identifier: 'box', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, - dependsOn: [ 'backup-app-00', 'backup-app-01' ], - manifest: null, - format: 'tgz' - }; - - var BACKUP_0_APP_0 = { // backup of installed app - id: 'backup-app-00', - identifier: APP.id, - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, - dependsOn: [], - manifest: null, - format: 'tgz' - }; - - var BACKUP_0_APP_1 = { // this app is uninstalled - id: 'backup-app-01', - identifier: 'app1', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, - dependsOn: [], - manifest: null, - format: 'tgz' - }; - - var BACKUP_1 = { - id: 'backup-box-1', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_BOX, - state: backups.BACKUP_STATE_NORMAL, - identifier: 'box', - dependsOn: [ 'backup-app-10', 'backup-app-11' ], - manifest: null, - format: 'tgz' - }; - - var BACKUP_1_APP_0 = { - id: 'backup-app-10', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, - identifier: APP.id, - dependsOn: [], - manifest: null, - format: 'tgz' - }; - - var BACKUP_1_APP_1 = { - id: 'backup-app-11', - encryptionVersion: null, - packageVersion: '1.0.0', - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_NORMAL, - identifier: 'app1', - dependsOn: [], - manifest: null, - format: 'tgz' - }; - - it('succeeds without backups', async function () { - cleanupBackups(); - }); - - it('succeeds with box backups, keeps latest', function (done) { - async.eachSeries([[ BACKUP_0, BACKUP_0_APP_0, BACKUP_0_APP_1 ], [ BACKUP_1, BACKUP_1_APP_0, BACKUP_1_APP_1 ]], function (backup, callback) { - // space out backups - setTimeout(function () { - async.eachSeries(backup, (b, done) => backupdb.add(b.id, b, done), callback); - }, 2000); - }, function (error) { - expect(error).to.not.be.ok(); - - util.callbackify(cleanupBackups)(function (error) { - expect(error).to.not.be.ok(); - - backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000, function (error, result) { - expect(error).to.not.be.ok(); - expect(result.length).to.equal(1); - expect(result[0].id).to.equal(BACKUP_1.id); - - // check that app backups are gone as well. only backup_1 will remain - backupdb.get(BACKUP_0_APP_0.id, function (error) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.equal(BoxError.NOT_FOUND); - - done(); - }); - }); - }); - }); - }); - - it('does not remove expired backups if only one left', function (done) { - util.callbackify(cleanupBackups)(function (error) { - expect(error).to.not.be.ok(); - - backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000, function (error, result) { - expect(error).to.not.be.ok(); - expect(result.length).to.equal(1); - expect(result[0].id).to.equal(BACKUP_1.id); - - // check that app backups are also still there. backup_1 is still there - backupdb.get(BACKUP_1_APP_0.id, function (error, result) { - expect(error).to.not.be.ok(); - expect(result.id).to.equal(BACKUP_1_APP_0.id); - - done(); - }); - }); - }); - }); - - it('succeeds for app backups not referenced by a box backup', function (done) { - // add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there - async.eachSeries([BACKUP_0_APP_0, BACKUP_0_APP_1], (b, done) => backupdb.add(b.id, b, done), function (error) { - expect(error).to.not.be.ok(); - - // wait for expiration - setTimeout(function () { - util.callbackify(cleanupBackups)(function (error) { - expect(error).to.not.be.ok(); - - backupdb.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000, function (error, result) { - expect(error).to.not.be.ok(); - expect(result.length).to.equal(3); - result = result.sort((r1, r2) => r1.id.localeCompare(r2.id)); - expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved - expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box - expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box - - done(); - }); - }); - }, 2000); - }); - }); + it('get of unknown id fails', async function () { + const result = await backups.get('somerandom'); + expect(result).to.be(null); }); - describe('fs meta data', function () { - let tmpdir; - before(function () { - tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test')); - }); - after(function () { - rimraf.sync(tmpdir); - }); - - it('saves special files', function (done) { - createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } }); - fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8)); - - let dataLayout = new DataLayout(tmpdir, []); - - backups._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, function (error) { - expect(error).to.not.be.ok(); - - var emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs; - expect(emptyDirs).to.eql(['./data/subdir/emptydir']); - - var execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles; - expect(execFiles).to.eql(['./dir2/file']); - - done(); - }); - }); - - it('restores special files', function (done) { - rimraf.sync(path.join(tmpdir, 'data')); - - expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked - - let dataLayout = new DataLayout(tmpdir, []); - - backups._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, function (error) { - expect(error).to.not.be.ok(); - - expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true); - var mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode; - expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8)); - - done(); - }); - }); + it('getByTypePaged succeeds', async function () { + const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5); + expect(results.length).to.be(1); + delete results[0].creationTime; + expect(results[0]).to.eql(boxBackup); }); - describe('filesystem', function () { - let backupInfo1; + it('delete succeeds', async function () { + await backups.del(boxBackup.id); + const result = await backups.get(boxBackup.id); + expect(result).to.be(null); + }); - const backupConfig = { - provider: 'filesystem', - backupFolder: path.join(os.tmpdir(), 'backups-test-filesystem'), - format: 'tgz', - retentionPolicy: { keepWithinSecs: 10000 }, - schedulePattern: '00 00 23 * * *' - }; + it('add app backup succeeds', async function () { + await backups.add(appBackup.id, appBackup); + }); - before(function (done) { - rimraf.sync(backupConfig.backupFolder); + it('get app backup succeeds', async function () { + const result = await backups.get(appBackup.id); + delete result.creationTime; + expect(result).to.eql(appBackup); + }); - done(); - }); + it('getByIdentifierAndStatePaged succeeds', async function () { + const results = await backups.getByIdentifierAndStatePaged(appBackup.identifier, backups.BACKUP_STATE_CREATING, 1, 5); + expect(results.length).to.be(1); + delete results[0].creationTime; + expect(results[0]).to.eql(appBackup); + }); - after(function (done) { - rimraf.sync(backupConfig.backupFolder); - done(); - }); - - it('fails to set backup config for bad folder', function (done) { - const tmp = Object.assign({}, backupConfig, { backupFolder: '/root/oof' }); - settings.setBackupConfig(tmp, function (error) { - expect(error).to.be.a(BoxError); - expect(error.reason).to.equal(BoxError.BAD_FIELD); - - done(); - }); - }); - - it('succeeds to set backup config', function (done) { - settings.setBackupConfig(backupConfig, function (error) { - expect(error).to.be(null); - - expect(fs.existsSync(path.join(backupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created - - done(); - }); - }); - - it('can backup', function (done) { - // arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/ - if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done(); - - createBackup(function (error, result) { - expect(error).to.be(null); - expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup - expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); - - backupInfo1 = result; - - done(); - }); - }); - - it('can take another backup', function (done) { - // arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/ - if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done(); - - createBackup(function (error, result) { - expect(error).to.be(null); - expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup - expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup - expect(fs.statSync(path.join(backupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore - - done(); - }); - }); + it('delete app backup succeeds', async function () { + await backups.del(appBackup.id); + const result = await backups.get(appBackup.id); + expect(result).to.be(null); }); }); diff --git a/src/test/backuptask-test.js b/src/test/backuptask-test.js new file mode 100644 index 000000000..25b6c79db --- /dev/null +++ b/src/test/backuptask-test.js @@ -0,0 +1,151 @@ +/* jslint node:true */ +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +'use strict'; + +const backups = require('../backups.js'), + backuptask = require('../backuptask.js'), + common = require('./common.js'), + DataLayout = require('../datalayout.js'), + delay = require('delay'), + expect = require('expect.js'), + fs = require('fs'), + os = require('os'), + path = require('path'), + safe = require('safetydance'), + settings = require('../settings.js'), + tasks = require('../tasks.js'); + +describe('backuptask', function () { + const { setup, cleanup, createTree } = common; + + before(setup); + after(cleanup); + + describe('fs meta data', function () { + let tmpdir; + before(function () { + tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test')); + }); + after(function () { + fs.rmSync(tmpdir, { recursive: true, force: true }); + }); + + it('saves special files', async function () { + createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } }); + fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8)); + + let dataLayout = new DataLayout(tmpdir, []); + + await backuptask._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); + + const emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs; + expect(emptyDirs).to.eql(['./data/subdir/emptydir']); + + const execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles; + expect(execFiles).to.eql(['./dir2/file']); + }); + + it('restores special files', async function () { + fs.rmSync(path.join(tmpdir, 'data'), { recursive: true, force: true }); + + expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked + + let dataLayout = new DataLayout(tmpdir, []); + + await backuptask._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); + + expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true); + const mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode; + expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8)); + }); + }); + + + describe('backupBoxAndApps', function () { + let backupInfo1; + + const backupConfig = { + provider: 'filesystem', + backupFolder: path.join(os.tmpdir(), 'backupstask-test-filesystem'), + format: 'tgz', + retentionPolicy: { keepWithinSecs: 10000 }, + schedulePattern: '00 00 23 * * *' + }; + + before(function (done) { + fs.rmSync(backupConfig.backupFolder, { recursive: true, force: true }); + + settings.setBackupConfig(backupConfig, done); + }); + + after(function () { + fs.rmSync(backupConfig.backupFolder, { recursive: true, force: true }); + }); + + function createBackup(callback) { + backups.startBackupTask({ username: 'test' }, async function (error, taskId) { // this call does not wait for the backup! + if (error) return callback(error); + + // eslint-disable-next-line no-constant-condition + while (true) { + await delay(1000); + + const p = await tasks.get(taskId); + + if (p.percent !== 100) continue; + if (p.error) return callback(new Error(`backup failed: ${p.error.message}`)); + if (!p.result) return callback(new Error('backup has no result:' + p)); + + const [error, result] = await safe(backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, 1, 1)); + + if (error) return callback(error); + if (result.length !== 1) return callback(new Error('result is not of length 1')); + + // the task progress and the db entry is set in the worker. wait for 2 seconds for backup lock to get released in parent process + await delay(2000); + + callback(null, result[0]); + } + }); + } + + it('can backup', function (done) { + // arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/ + if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) { + console.log('test skipped because of MariaDB'); + return done(); + } + + createBackup(function (error, result) { + expect(error).to.be(null); + expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup + expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); + + backupInfo1 = result; + + done(); + }); + }); + + it('can take another backup', function (done) { + // arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/ + if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) { + console.log('test skipped because of MariaDB'); + return done(); + } + + createBackup(function (error, result) { + expect(error).to.be(null); + expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup + expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup + expect(fs.statSync(path.join(backupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore + + done(); + }); + }); + }); +}); diff --git a/src/test/common.js b/src/test/common.js index 363f3f7e7..96069a824 100644 --- a/src/test/common.js +++ b/src/test/common.js @@ -17,6 +17,7 @@ const appdb = require('../appdb.js'), rimraf = require('rimraf'), settings = require('../settings.js'), settingsdb = require('../settingsdb.js'), + tasks = require('../tasks.js'), userdb = require('../userdb.js'), users = require('../users.js'); @@ -182,6 +183,7 @@ function setup(done) { (done) => mailboxdb.addMailbox(exports.MAILBOX_NAME, DOMAIN.domain, { ownerId: USER.id, ownerType: mail.OWNERTYPE_USER, active: true }, done), (done) => mailboxdb.setAliasesForName(exports.MAILBOX_NAME, DOMAIN.domain, [ { name: exports.ALIAS_NAME, domain: DOMAIN.domain} ], done), + tasks.stopAllTasks, ], done); } diff --git a/src/test/storage-test.js b/src/test/storage-test.js index cfc8fc899..caa0f8ffc 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -5,7 +5,8 @@ 'use strict'; -var BoxError = require('../boxerror.js'), +const BoxError = require('../boxerror.js'), + common = require('./common.js'), execSync = require('child_process').execSync, expect = require('expect.js'), filesystem = require('../storage/filesystem.js'), @@ -17,19 +18,26 @@ var BoxError = require('../boxerror.js'), rimraf = require('rimraf'), recursive_readdir = require('recursive-readdir'), s3 = require('../storage/s3.js'), + settings = require('../settings.js'), gcs = require('../storage/gcs.js'), chunk = require('lodash.chunk'); describe('Storage', function () { + const { setup, cleanup } = common; + + before(setup); + after(cleanup); + describe('filesystem', function () { + let gTmpFolder; - var gTmpFolder; - - var gBackupConfig = { + const gBackupConfig = { provider: 'filesystem', key: 'key', backupFolder: null, - format: 'tgz' + format: 'tgz', + retentionPolicy: { keepWithinSecs: 10000 }, + schedulePattern: '00 00 23 * * *' }; before(function (done) { @@ -45,6 +53,26 @@ describe('Storage', function () { done(); }); + it('fails to set backup config for bad folder', function (done) { + const tmp = Object.assign({}, gBackupConfig, { backupFolder: '/root/oof' }); + settings.setBackupConfig(tmp, function (error) { + expect(error).to.be.a(BoxError); + expect(error.reason).to.equal(BoxError.BAD_FIELD); + + done(); + }); + }); + + it('succeeds to set backup config', function (done) { + settings.setBackupConfig(gBackupConfig, function (error) { + expect(error).to.be(null); + + expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created + + done(); + }); + }); + it('can upload', function (done) { var sourceFile = path.join(__dirname, 'storage/data/test.txt'); var sourceStream = fs.createReadStream(sourceFile);