/* jslint node:true */ /* global it:false */ /* global describe:false */ /* global before:false */ /* global after:false */ 'use strict'; const async = require('async'), backupdb = require('../backupdb.js'), backups = require('../backups.js'), BoxError = require('../boxerror.js'), common = require('./common.js'), DataLayout = require('../datalayout.js'), delay = require('delay'), expect = require('expect.js'), fs = require('fs'), os = require('os'), moment = require('moment'), path = require('path'), rimraf = require('rimraf'), safe = require('safetydance'), settingsdb = require('../settingsdb.js'), settings = require('../settings.js'), tasks = require('../tasks.js'), util = require('util'); const { createTree, APP } = common; function createBackup(callback) { backups.startBackupTask({ username: 'test' }, function (error, taskId) { // this call does not wait for the backup! if (error) return callback(error); async function waitForBackup() { const [error, p] = await safe(tasks.get(taskId)); if (error) return callback(error); if (p.percent !== 100) return setTimeout(waitForBackup, 1000); if (p.errorMessage) return callback(new Error('backup failed:' + p)); if (!p.result) return callback(new Error('backup has no result:' + p)); backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, 1, 1, function (error, result) { if (error) return callback(error); if (result.length !== 1) return callback(new Error('result is not of length 1')); // the task progress and the db entry is set in the worker. wait for 2 seconds for backup lock to get released in parent process setTimeout(() => callback(null, result[0]), 2000); }); } setTimeout(waitForBackup, 1000); }); } async function cleanupBackups() { const taskId = await backups.startCleanupTask({ username: 'test' }); // eslint-disable-next-line no-constant-condition while (true) { await delay(1000); const p = await tasks.get(taskId); if (p.percent !== 100) continue; if (p.errorMessage) throw new Error('backup failed:' + p.errorMessage); return; } } describe('backups', function () { before(function (done) { const BACKUP_DIR = path.join(os.tmpdir(), 'cloudron-backup-test'); async.series([ common.setup, fs.mkdir.bind(null, BACKUP_DIR, { recursive: true }), settingsdb.set.bind(null, settings.BACKUP_CONFIG_KEY, JSON.stringify({ provider: 'filesystem', password: 'supersecret', backupFolder: BACKUP_DIR, retentionPolicy: { keepWithinSecs: 1 }, format: 'tgz' })) ], done); }); after(common.cleanup); describe('retention policy', function () { it('keeps latest', function () { let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); expect(backup.keepReason).to.be('latest'); }); it('does not keep latest', function () { let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []); expect(backup.keepReason).to.be(undefined); }); it('always keeps forever policy', function () { let backup = { creationTime: new Date() }; backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []); expect(backup.keepReason).to.be('keepWithinSecs'); }); it('preserveSecs takes precedence', function () { let backup = { creationTime: new Date(), preserveSecs: 3000 }; backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); expect(backup.keepReason).to.be('preserveSecs'); }); it('1 daily', function () { let b = [ { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; backups._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('keepDaily'); expect(b[1].keepReason).to.be(undefined); expect(b[2].keepReason).to.be(undefined); expect(b[3].keepReason).to.be(undefined); expect(b[3].keepReason).to.be(undefined); }); // if you are debugging this test, it's because of some timezone issue with all the hour substraction! it('2 daily, 1 weekly', function () { let b = [ { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, { id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() }, { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); expect(b[0].keepReason).to.be('keepDaily'); // today expect(b[1].keepReason).to.be('keepWeekly'); // today expect(b[2].keepReason).to.be(undefined); expect(b[3].keepReason).to.be('keepDaily'); // yesterday expect(b[4].discardReason).to.be('error'); // errored expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long expect(b[6].keepReason).to.be(undefined); // outside retention policy }); it('2 daily, 3 monthly, 1 yearly', function () { let b = [ { id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() }, { id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() }, { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, { id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, { id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() }, { id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() }, { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, ]; backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('creating'); expect(b[1].discardReason).to.be('error'); // errored expect(b[2].keepReason).to.be('keepDaily'); expect(b[3].keepReason).to.be('keepMonthly'); expect(b[4].keepReason).to.be('keepYearly'); expect(b[5].keepReason).to.be('keepDaily'); // yesterday expect(b[6].discardReason).to.be('creating-too-long'); // errored expect(b[7].keepReason).to.be('keepMonthly'); expect(b[8].keepReason).to.be('keepMonthly'); expect(b[9].keepReason).to.be(undefined); }); }); describe('cleanup', function () { var BACKUP_0 = { id: 'backup-box-0', identifier: 'box', encryptionVersion: null, packageVersion: '1.0.0', type: backups.BACKUP_TYPE_BOX, state: backups.BACKUP_STATE_NORMAL, dependsOn: [ 'backup-app-00', 'backup-app-01' ], manifest: null, format: 'tgz' }; var BACKUP_0_APP_0 = { // backup of installed app id: 'backup-app-00', identifier: APP.id, encryptionVersion: null, packageVersion: '1.0.0', type: backups.BACKUP_TYPE_APP, state: backups.BACKUP_STATE_NORMAL, dependsOn: [], manifest: null, format: 'tgz' }; var BACKUP_0_APP_1 = { // this app is uninstalled id: 'backup-app-01', identifier: 'app1', encryptionVersion: null, packageVersion: '1.0.0', type: backups.BACKUP_TYPE_APP, state: backups.BACKUP_STATE_NORMAL, dependsOn: [], manifest: null, format: 'tgz' }; var BACKUP_1 = { id: 'backup-box-1', encryptionVersion: null, packageVersion: '1.0.0', type: backups.BACKUP_TYPE_BOX, state: backups.BACKUP_STATE_NORMAL, identifier: 'box', dependsOn: [ 'backup-app-10', 'backup-app-11' ], manifest: null, format: 'tgz' }; var BACKUP_1_APP_0 = { id: 'backup-app-10', encryptionVersion: null, packageVersion: '1.0.0', type: backups.BACKUP_TYPE_APP, state: backups.BACKUP_STATE_NORMAL, identifier: APP.id, dependsOn: [], manifest: null, format: 'tgz' }; var BACKUP_1_APP_1 = { id: 'backup-app-11', encryptionVersion: null, packageVersion: '1.0.0', type: backups.BACKUP_TYPE_APP, state: backups.BACKUP_STATE_NORMAL, identifier: 'app1', dependsOn: [], manifest: null, format: 'tgz' }; it('succeeds without backups', async function () { cleanupBackups(); }); it('succeeds with box backups, keeps latest', function (done) { async.eachSeries([[ BACKUP_0, BACKUP_0_APP_0, BACKUP_0_APP_1 ], [ BACKUP_1, BACKUP_1_APP_0, BACKUP_1_APP_1 ]], function (backup, callback) { // space out backups setTimeout(function () { async.eachSeries(backup, (b, done) => backupdb.add(b.id, b, done), callback); }, 2000); }, function (error) { expect(error).to.not.be.ok(); util.callbackify(cleanupBackups)(function (error) { expect(error).to.not.be.ok(); backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000, function (error, result) { expect(error).to.not.be.ok(); expect(result.length).to.equal(1); expect(result[0].id).to.equal(BACKUP_1.id); // check that app backups are gone as well. only backup_1 will remain backupdb.get(BACKUP_0_APP_0.id, function (error) { expect(error).to.be.a(BoxError); expect(error.reason).to.equal(BoxError.NOT_FOUND); done(); }); }); }); }); }); it('does not remove expired backups if only one left', function (done) { util.callbackify(cleanupBackups)(function (error) { expect(error).to.not.be.ok(); backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000, function (error, result) { expect(error).to.not.be.ok(); expect(result.length).to.equal(1); expect(result[0].id).to.equal(BACKUP_1.id); // check that app backups are also still there. backup_1 is still there backupdb.get(BACKUP_1_APP_0.id, function (error, result) { expect(error).to.not.be.ok(); expect(result.id).to.equal(BACKUP_1_APP_0.id); done(); }); }); }); }); it('succeeds for app backups not referenced by a box backup', function (done) { // add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there async.eachSeries([BACKUP_0_APP_0, BACKUP_0_APP_1], (b, done) => backupdb.add(b.id, b, done), function (error) { expect(error).to.not.be.ok(); // wait for expiration setTimeout(function () { util.callbackify(cleanupBackups)(function (error) { expect(error).to.not.be.ok(); backupdb.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000, function (error, result) { expect(error).to.not.be.ok(); expect(result.length).to.equal(3); result = result.sort((r1, r2) => r1.id.localeCompare(r2.id)); expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box done(); }); }); }, 2000); }); }); }); describe('fs meta data', function () { let tmpdir; before(function () { tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test')); }); after(function () { rimraf.sync(tmpdir); }); it('saves special files', function (done) { createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } }); fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8)); let dataLayout = new DataLayout(tmpdir, []); backups._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, function (error) { expect(error).to.not.be.ok(); var emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs; expect(emptyDirs).to.eql(['./data/subdir/emptydir']); var execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles; expect(execFiles).to.eql(['./dir2/file']); done(); }); }); it('restores special files', function (done) { rimraf.sync(path.join(tmpdir, 'data')); expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked let dataLayout = new DataLayout(tmpdir, []); backups._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, function (error) { expect(error).to.not.be.ok(); expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true); var mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode; expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8)); done(); }); }); }); describe('filesystem', function () { let backupInfo1; const backupConfig = { provider: 'filesystem', backupFolder: path.join(os.tmpdir(), 'backups-test-filesystem'), format: 'tgz', retentionPolicy: { keepWithinSecs: 10000 }, schedulePattern: '00 00 23 * * *' }; before(function (done) { rimraf.sync(backupConfig.backupFolder); done(); }); after(function (done) { rimraf.sync(backupConfig.backupFolder); done(); }); it('fails to set backup config for bad folder', function (done) { const tmp = Object.assign({}, backupConfig, { backupFolder: '/root/oof' }); settings.setBackupConfig(tmp, function (error) { expect(error).to.be.a(BoxError); expect(error.reason).to.equal(BoxError.BAD_FIELD); done(); }); }); it('succeeds to set backup config', function (done) { settings.setBackupConfig(backupConfig, function (error) { expect(error).to.be(null); expect(fs.existsSync(path.join(backupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created done(); }); }); it('can backup', function (done) { // arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/ if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done(); createBackup(function (error, result) { expect(error).to.be(null); expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); backupInfo1 = result; done(); }); }); it('can take another backup', function (done) { // arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/ if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done(); createBackup(function (error, result) { expect(error).to.be(null); expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup expect(fs.statSync(path.join(backupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore done(); }); }); }); });