jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix the backup tests

Browse Source
This commit is contained in:
Girish Ramakrishnan
2021-07-14 19:03:12 -07:00
parent 004e812d60
commit 2840bba4bf
8 changed files with 585 additions and 623 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
src/test/backupcleaner-test.js Normal file
View File

@@ -0,0 +1,280 @@
/* jslint node:true */
/* global it:false */
/* global describe:false */
/* global before:false */
/* global after:false */
'use strict';
const backupCleaner = require('../backupcleaner.js'),
backups = require('../backups.js'),
common = require('./common.js'),
delay = require('delay'),
expect = require('expect.js'),
moment = require('moment'),
settings = require('../settings.js'),
settingsdb = require('../settingsdb.js'),
tasks = require('../tasks.js');
describe('backup cleaner', function () {
const { setup, cleanup, APP } = common;
before(setup);
after(cleanup);
const backupTemplate = {
id: 'someid',
encryptionVersion: 2,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'box',
dependsOn: [ 'dep1' ],
manifest: null,
format: 'tgz',
preserveSecs: 0
};
describe('retention policy', function () {
it('keeps latest', function () {
const backup = Object.assign({}, backupTemplate, { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL });
backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []);
expect(backup.keepReason).to.be('latest');
});
it('does not keep latest', function () {
let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL };
backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []);
expect(backup.keepReason).to.be(undefined);
});
it('always keeps forever policy', function () {
let backup = { creationTime: new Date() };
backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []);
expect(backup.keepReason).to.be('keepWithinSecs');
});
it('preserveSecs takes precedence', function () {
let backup = { creationTime: new Date(), preserveSecs: 3000 };
backupCleaner._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []);
expect(backup.keepReason).to.be('preserveSecs');
});
it('1 daily', function () {
let b = [
{ id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
];
backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []);
expect(b[0].keepReason).to.be('keepDaily');
expect(b[1].keepReason).to.be(undefined);
expect(b[2].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be(undefined);
});
// if you are debugging this test, it's because of some timezone issue with all the hour substraction!
it('2 daily, 1 weekly', function () {
let b = [
{ id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() },
{ id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() },
{ id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
];
backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []);
expect(b[0].keepReason).to.be('keepDaily'); // today
expect(b[1].keepReason).to.be('keepWeekly'); // today
expect(b[2].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be('keepDaily'); // yesterday
expect(b[4].discardReason).to.be('error'); // errored
expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long
expect(b[6].keepReason).to.be(undefined); // outside retention policy
});
it('2 daily, 3 monthly, 1 yearly', function () {
let b = [
{ id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
{ id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() },
{ id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() },
{ id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() },
{ id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() },
];
backupCleaner._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []);
expect(b[0].keepReason).to.be('creating');
expect(b[1].discardReason).to.be('error'); // errored
expect(b[2].keepReason).to.be('keepDaily');
expect(b[3].keepReason).to.be('keepMonthly');
expect(b[4].keepReason).to.be('keepYearly');
expect(b[5].keepReason).to.be('keepDaily'); // yesterday
expect(b[6].discardReason).to.be('creating-too-long'); // errored
expect(b[7].keepReason).to.be('keepMonthly');
expect(b[8].keepReason).to.be('keepMonthly');
expect(b[9].keepReason).to.be(undefined);
});
});
describe('task', function () {
const BACKUP_0_BOX = {
id: 'backup-box-0',
identifier: 'box',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
dependsOn: [ 'backup-app-00', 'backup-app-01' ],
manifest: null,
format: 'tgz'
};
const BACKUP_0_APP_0 = { // backup of installed app
id: 'backup-app-00',
identifier: APP.id,
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
dependsOn: [],
manifest: null,
format: 'tgz'
};
const BACKUP_0_APP_1 = { // this app is uninstalled
id: 'backup-app-01',
identifier: 'app1',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
dependsOn: [],
manifest: null,
format: 'tgz'
};
const BACKUP_1_BOX = {
id: 'backup-box-1',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'box',
dependsOn: [ 'backup-app-10', 'backup-app-11' ],
manifest: null,
format: 'tgz'
};
const BACKUP_1_APP_0 = {
id: 'backup-app-10',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
identifier: APP.id,
dependsOn: [],
manifest: null,
format: 'tgz'
};
const BACKUP_1_APP_1 = {
id: 'backup-app-11',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'app1',
dependsOn: [],
manifest: null,
format: 'tgz'
};
before(function (done) {
settingsdb.set(settings.BACKUP_CONFIG_KEY, JSON.stringify({
provider: 'filesystem',
password: 'supersecret',
backupFolder: '/tmp/someplace',
retentionPolicy: { keepWithinSecs: 1 },
format: 'tgz'
}), done);
});
async function cleanupBackups() {
const taskId = await backups.startCleanupTask({ username: 'test' });
console.log('started task', taskId);
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(1000);
const p = await tasks.get(taskId);
if (p.percent !== 100) continue;
if (p.error) throw new Error(`backup failed: ${p.error.message}`);
return;
}
}
it('succeeds without backups', async function () {
await cleanupBackups();
});
it('succeeds with box backups, keeps latest', async function () {
for (const backup of [[ BACKUP_0_BOX, BACKUP_0_APP_0, BACKUP_0_APP_1 ], [ BACKUP_1_BOX, BACKUP_1_APP_0, BACKUP_1_APP_1 ]]) {
await delay(2000); // space out backups
for (const b of backup) {
await backups.add(b.id, b);
}
}
await cleanupBackups();
const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
expect(results.length).to.equal(1);
expect(results[0].id).to.equal(BACKUP_1_BOX.id);
// check that app backups are gone as well. only backup_1 will remain
const result = await backups.get(BACKUP_0_APP_0.id);
expect(result).to.be(null);
});
it('does not remove expired backups if only one left', async function () {
await cleanupBackups();
const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
expect(results[0].id).to.equal(BACKUP_1_BOX.id);
// check that app backups are also still there. backup_1 is still there
const result = await backups.get(BACKUP_1_APP_0.id);
expect(result.id).to.equal(BACKUP_1_APP_0.id);
});
it('succeeds for app backups not referenced by a box backup', async function () {
// add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there
for (const backup of [BACKUP_0_APP_0, BACKUP_0_APP_1]) {
await backups.add(backup.id, backup);
}
await delay(2000); // wait for expiration
await cleanupBackups();
let result = await backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000);
expect(result.length).to.equal(3);
result = result.sort((r1, r2) => r1.id.localeCompare(r2.id));
expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved
expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box
expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box
});
});
});

632
src/test/backups-test.js
View File

@@ -6,585 +6,97 @@
'use strict';
const async = require('async'),
backupdb = require('../backupdb.js'),
backups = require('../backups.js'),
const backups = require('../backups.js'),
BoxError = require('../boxerror.js'),
common = require('./common.js'),
DataLayout = require('../datalayout.js'),
delay = require('delay'),
expect = require('expect.js'),
fs = require('fs'),
os = require('os'),
moment = require('moment'),
path = require('path'),
rimraf = require('rimraf'),
safe = require('safetydance'),
settingsdb = require('../settingsdb.js'),
settings = require('../settings.js'),
tasks = require('../tasks.js'),
util = require('util');
const { createTree, APP } = common;
function createBackup(callback) {
backups.startBackupTask({ username: 'test' }, function (error, taskId) { // this call does not wait for the backup!
if (error) return callback(error);
async function waitForBackup() {
const [error, p] = await safe(tasks.get(taskId));
if (error) return callback(error);
if (p.percent !== 100) return setTimeout(waitForBackup, 1000);
if (p.errorMessage) return callback(new Error('backup failed:' + p));
if (!p.result) return callback(new Error('backup has no result:' + p));
backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, 1, 1, function (error, result) {
if (error) return callback(error);
if (result.length !== 1) return callback(new Error('result is not of length 1'));
// the task progress and the db entry is set in the worker. wait for 2 seconds for backup lock to get released in parent process
setTimeout(() => callback(null, result[0]), 2000);
});
}
setTimeout(waitForBackup, 1000);
});
}
async function cleanupBackups() {
const taskId = await backups.startCleanupTask({ username: 'test' });
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(1000);
const p = await tasks.get(taskId);
if (p.percent !== 100) continue;
if (p.errorMessage) throw new Error('backup failed:' + p.errorMessage);
return;
}
}
safe = require('safetydance');
describe('backups', function () {
before(function (done) {
const BACKUP_DIR = path.join(os.tmpdir(), 'cloudron-backup-test');
const { setup, cleanup } = common;
async.series([
common.setup,
fs.mkdir.bind(null, BACKUP_DIR, { recursive: true }),
settingsdb.set.bind(null, settings.BACKUP_CONFIG_KEY, JSON.stringify({
provider: 'filesystem',
password: 'supersecret',
backupFolder: BACKUP_DIR,
retentionPolicy: { keepWithinSecs: 1 },
format: 'tgz'
}))
], done);
before(setup);
after(cleanup);
const boxBackup = {
id: 'backup-box',
encryptionVersion: 2,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'box',
dependsOn: [ 'dep1' ],
manifest: null,
format: 'tgz',
preserveSecs: 0
};
const appBackup = {
id: 'app_appid_123',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_CREATING,
identifier: 'appid',
dependsOn: [ ],
manifest: { foo: 'bar' },
format: 'tgz',
preserveSecs: 0
};
it('add succeeds', async function () {
await backups.add(boxBackup.id, boxBackup);
});
after(common.cleanup);
describe('backup', function () {
it('add succeeds', function (done) {
var backup = {
id: 'backup-box',
encryptionVersion: 2,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'box',
dependsOn: [ 'dep1' ],
manifest: null,
format: 'tgz'
};
backupdb.add(backup.id, backup, function (error) {
expect(error).to.be(null);
done();
});
});
it('get succeeds', function (done) {
backupdb.get('backup-box', function (error, result) {
expect(error).to.be(null);
expect(result.encryptionVersion).to.be(2);
expect(result.packageVersion).to.be('1.0.0');
expect(result.type).to.be(backups.BACKUP_TYPE_BOX);
expect(result.state).to.be(backups.BACKUP_STATE_NORMAL);
expect(result.creationTime).to.be.a(Date);
expect(result.dependsOn).to.eql(['dep1']);
expect(result.manifest).to.eql(null);
done();
});
});
it('get of unknown id fails', function (done) {
backupdb.get('somerandom', function (error, result) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.be(BoxError.NOT_FOUND);
expect(result).to.not.be.ok();
done();
});
});
it('getByTypePaged succeeds', function (done) {
backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5, function (error, results) {
expect(error).to.be(null);
expect(results).to.be.an(Array);
expect(results.length).to.be(1);
expect(results[0].id).to.be('backup-box');
expect(results[0].encryptionVersion).to.be(2);
expect(results[0].packageVersion).to.be('1.0.0');
expect(results[0].dependsOn).to.eql(['dep1']);
expect(results[0].manifest).to.eql(null);
done();
});
});
it('delete succeeds', function (done) {
backupdb.del('backup-box', function (error, result) {
expect(error).to.be(null);
expect(result).to.not.be.ok();
backupdb.get('backup-box', function (error, result) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.NOT_FOUND);
expect(result).to.not.be.ok();
done();
});
});
});
it('add app succeeds', function (done) {
var backup = {
id: 'app_appid_123',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_CREATING,
identifier: 'appid',
dependsOn: [ ],
manifest: { foo: 'bar' },
format: 'tgz'
};
backupdb.add(backup.id, backup, function (error) {
expect(error).to.be(null);
done();
});
});
it('get succeeds', function (done) {
backupdb.get('app_appid_123', function (error, result) {
expect(error).to.be(null);
expect(result.encryptionVersion).to.be(null);
expect(result.packageVersion).to.be('1.0.0');
expect(result.type).to.be(backups.BACKUP_TYPE_APP);
expect(result.state).to.be(backups.BACKUP_STATE_CREATING);
expect(result.creationTime).to.be.a(Date);
expect(result.dependsOn).to.eql([]);
expect(result.manifest).to.eql({ foo: 'bar' });
done();
});
});
it('getByIdentifierPaged succeeds', function (done) {
backupdb.getByIdentifierPaged('appid', 1, 5, function (error, results) {
expect(error).to.be(null);
expect(results).to.be.an(Array);
expect(results.length).to.be(1);
expect(results[0].id).to.be('app_appid_123');
expect(results[0].encryptionVersion).to.be(null);
expect(results[0].packageVersion).to.be('1.0.0');
expect(results[0].dependsOn).to.eql([]);
expect(results[0].manifest).to.eql({ foo: 'bar' });
done();
});
});
it('delete succeeds', function (done) {
backupdb.del('app_appid_123', function (error, result) {
expect(error).to.be(null);
expect(result).to.not.be.ok();
backupdb.get('app_appid_123', function (error, result) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.NOT_FOUND);
expect(result).to.not.be.ok();
done();
});
});
});
it('fails with duplicating id', async function () {
const [error] = await safe(backups.add(boxBackup.id, boxBackup));
expect(error.reason).to.be(BoxError.ALREADY_EXISTS);
});
describe('retention policy', function () {
it('keeps latest', function () {
let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []);
expect(backup.keepReason).to.be('latest');
});
it('does not keep latest', function () {
let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []);
expect(backup.keepReason).to.be(undefined);
});
it('always keeps forever policy', function () {
let backup = { creationTime: new Date() };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []);
expect(backup.keepReason).to.be('keepWithinSecs');
});
it('preserveSecs takes precedence', function () {
let backup = { creationTime: new Date(), preserveSecs: 3000 };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []);
expect(backup.keepReason).to.be('preserveSecs');
});
it('1 daily', function () {
let b = [
{ id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
];
backups._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []);
expect(b[0].keepReason).to.be('keepDaily');
expect(b[1].keepReason).to.be(undefined);
expect(b[2].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be(undefined);
});
// if you are debugging this test, it's because of some timezone issue with all the hour substraction!
it('2 daily, 1 weekly', function () {
let b = [
{ id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() },
{ id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() },
{ id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
];
backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []);
expect(b[0].keepReason).to.be('keepDaily'); // today
expect(b[1].keepReason).to.be('keepWeekly'); // today
expect(b[2].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be('keepDaily'); // yesterday
expect(b[4].discardReason).to.be('error'); // errored
expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long
expect(b[6].keepReason).to.be(undefined); // outside retention policy
});
it('2 daily, 3 monthly, 1 yearly', function () {
let b = [
{ id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
{ id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() },
{ id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() },
{ id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() },
{ id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() },
];
backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []);
expect(b[0].keepReason).to.be('creating');
expect(b[1].discardReason).to.be('error'); // errored
expect(b[2].keepReason).to.be('keepDaily');
expect(b[3].keepReason).to.be('keepMonthly');
expect(b[4].keepReason).to.be('keepYearly');
expect(b[5].keepReason).to.be('keepDaily'); // yesterday
expect(b[6].discardReason).to.be('creating-too-long'); // errored
expect(b[7].keepReason).to.be('keepMonthly');
expect(b[8].keepReason).to.be('keepMonthly');
expect(b[9].keepReason).to.be(undefined);
});
it('get succeeds', async function () {
const result = await backups.get(boxBackup.id);
delete result.creationTime;
expect(result).to.eql(boxBackup);
});
describe('cleanup', function () {
var BACKUP_0 = {
id: 'backup-box-0',
identifier: 'box',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
dependsOn: [ 'backup-app-00', 'backup-app-01' ],
manifest: null,
format: 'tgz'
};
var BACKUP_0_APP_0 = { // backup of installed app
id: 'backup-app-00',
identifier: APP.id,
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
dependsOn: [],
manifest: null,
format: 'tgz'
};
var BACKUP_0_APP_1 = { // this app is uninstalled
id: 'backup-app-01',
identifier: 'app1',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
dependsOn: [],
manifest: null,
format: 'tgz'
};
var BACKUP_1 = {
id: 'backup-box-1',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'box',
dependsOn: [ 'backup-app-10', 'backup-app-11' ],
manifest: null,
format: 'tgz'
};
var BACKUP_1_APP_0 = {
id: 'backup-app-10',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
identifier: APP.id,
dependsOn: [],
manifest: null,
format: 'tgz'
};
var BACKUP_1_APP_1 = {
id: 'backup-app-11',
encryptionVersion: null,
packageVersion: '1.0.0',
type: backups.BACKUP_TYPE_APP,
state: backups.BACKUP_STATE_NORMAL,
identifier: 'app1',
dependsOn: [],
manifest: null,
format: 'tgz'
};
it('succeeds without backups', async function () {
cleanupBackups();
});
it('succeeds with box backups, keeps latest', function (done) {
async.eachSeries([[ BACKUP_0, BACKUP_0_APP_0, BACKUP_0_APP_1 ], [ BACKUP_1, BACKUP_1_APP_0, BACKUP_1_APP_1 ]], function (backup, callback) {
// space out backups
setTimeout(function () {
async.eachSeries(backup, (b, done) => backupdb.add(b.id, b, done), callback);
}, 2000);
}, function (error) {
expect(error).to.not.be.ok();
util.callbackify(cleanupBackups)(function (error) {
expect(error).to.not.be.ok();
backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000, function (error, result) {
expect(error).to.not.be.ok();
expect(result.length).to.equal(1);
expect(result[0].id).to.equal(BACKUP_1.id);
// check that app backups are gone as well. only backup_1 will remain
backupdb.get(BACKUP_0_APP_0.id, function (error) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.NOT_FOUND);
done();
});
});
});
});
});
it('does not remove expired backups if only one left', function (done) {
util.callbackify(cleanupBackups)(function (error) {
expect(error).to.not.be.ok();
backupdb.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000, function (error, result) {
expect(error).to.not.be.ok();
expect(result.length).to.equal(1);
expect(result[0].id).to.equal(BACKUP_1.id);
// check that app backups are also still there. backup_1 is still there
backupdb.get(BACKUP_1_APP_0.id, function (error, result) {
expect(error).to.not.be.ok();
expect(result.id).to.equal(BACKUP_1_APP_0.id);
done();
});
});
});
});
it('succeeds for app backups not referenced by a box backup', function (done) {
// add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there
async.eachSeries([BACKUP_0_APP_0, BACKUP_0_APP_1], (b, done) => backupdb.add(b.id, b, done), function (error) {
expect(error).to.not.be.ok();
// wait for expiration
setTimeout(function () {
util.callbackify(cleanupBackups)(function (error) {
expect(error).to.not.be.ok();
backupdb.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000, function (error, result) {
expect(error).to.not.be.ok();
expect(result.length).to.equal(3);
result = result.sort((r1, r2) => r1.id.localeCompare(r2.id));
expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved
expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box
expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box
done();
});
});
}, 2000);
});
});
it('get of unknown id fails', async function () {
const result = await backups.get('somerandom');
expect(result).to.be(null);
});
describe('fs meta data', function () {
let tmpdir;
before(function () {
tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test'));
});
after(function () {
rimraf.sync(tmpdir);
});
it('saves special files', function (done) {
createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } });
fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8));
let dataLayout = new DataLayout(tmpdir, []);
backups._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, function (error) {
expect(error).to.not.be.ok();
var emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs;
expect(emptyDirs).to.eql(['./data/subdir/emptydir']);
var execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles;
expect(execFiles).to.eql(['./dir2/file']);
done();
});
});
it('restores special files', function (done) {
rimraf.sync(path.join(tmpdir, 'data'));
expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked
let dataLayout = new DataLayout(tmpdir, []);
backups._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`, function (error) {
expect(error).to.not.be.ok();
expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true);
var mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode;
expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8));
done();
});
});
it('getByTypePaged succeeds', async function () {
const results = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 5);
expect(results.length).to.be(1);
delete results[0].creationTime;
expect(results[0]).to.eql(boxBackup);
});
describe('filesystem', function () {
let backupInfo1;
it('delete succeeds', async function () {
await backups.del(boxBackup.id);
const result = await backups.get(boxBackup.id);
expect(result).to.be(null);
});
const backupConfig = {
provider: 'filesystem',
backupFolder: path.join(os.tmpdir(), 'backups-test-filesystem'),
format: 'tgz',
retentionPolicy: { keepWithinSecs: 10000 },
schedulePattern: '00 00 23 * * *'
};
it('add app backup succeeds', async function () {
await backups.add(appBackup.id, appBackup);
});
before(function (done) {
rimraf.sync(backupConfig.backupFolder);
it('get app backup succeeds', async function () {
const result = await backups.get(appBackup.id);
delete result.creationTime;
expect(result).to.eql(appBackup);
});
done();
});
it('getByIdentifierAndStatePaged succeeds', async function () {
const results = await backups.getByIdentifierAndStatePaged(appBackup.identifier, backups.BACKUP_STATE_CREATING, 1, 5);
expect(results.length).to.be(1);
delete results[0].creationTime;
expect(results[0]).to.eql(appBackup);
});
after(function (done) {
rimraf.sync(backupConfig.backupFolder);
done();
});
it('fails to set backup config for bad folder', function (done) {
const tmp = Object.assign({}, backupConfig, { backupFolder: '/root/oof' });
settings.setBackupConfig(tmp, function (error) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.BAD_FIELD);
done();
});
});
it('succeeds to set backup config', function (done) {
settings.setBackupConfig(backupConfig, function (error) {
expect(error).to.be(null);
expect(fs.existsSync(path.join(backupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created
done();
});
});
it('can backup', function (done) {
// arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/
if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done();
createBackup(function (error, result) {
expect(error).to.be(null);
expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2);
backupInfo1 = result;
done();
});
});
it('can take another backup', function (done) {
// arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/
if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) return done();
createBackup(function (error, result) {
expect(error).to.be(null);
expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore
done();
});
});
it('delete app backup succeeds', async function () {
await backups.del(appBackup.id);
const result = await backups.get(appBackup.id);
expect(result).to.be(null);
});
});

151
src/test/backuptask-test.js Normal file
View File

@@ -0,0 +1,151 @@
/* jslint node:true */
/* global it:false */
/* global describe:false */
/* global before:false */
/* global after:false */
'use strict';
const backups = require('../backups.js'),
backuptask = require('../backuptask.js'),
common = require('./common.js'),
DataLayout = require('../datalayout.js'),
delay = require('delay'),
expect = require('expect.js'),
fs = require('fs'),
os = require('os'),
path = require('path'),
safe = require('safetydance'),
settings = require('../settings.js'),
tasks = require('../tasks.js');
describe('backuptask', function () {
const { setup, cleanup, createTree } = common;
before(setup);
after(cleanup);
describe('fs meta data', function () {
let tmpdir;
before(function () {
tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), 'backups-test'));
});
after(function () {
fs.rmSync(tmpdir, { recursive: true, force: true });
});
it('saves special files', async function () {
createTree(tmpdir, { 'data': { 'subdir': { 'emptydir': { } } }, 'dir2': { 'file': 'stuff' } });
fs.chmodSync(path.join(tmpdir, 'dir2/file'), parseInt('0755', 8));
let dataLayout = new DataLayout(tmpdir, []);
await backuptask._saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
const emptyDirs = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).emptyDirs;
expect(emptyDirs).to.eql(['./data/subdir/emptydir']);
const execFiles = JSON.parse(fs.readFileSync(path.join(tmpdir, 'fsmetadata.json'), 'utf8')).execFiles;
expect(execFiles).to.eql(['./dir2/file']);
});
it('restores special files', async function () {
fs.rmSync(path.join(tmpdir, 'data'), { recursive: true, force: true });
expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(false); // just make sure rimraf worked
let dataLayout = new DataLayout(tmpdir, []);
await backuptask._restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
expect(fs.existsSync(path.join(tmpdir, 'data/subdir/emptydir'))).to.be(true);
const mode = fs.statSync(path.join(tmpdir, 'dir2/file')).mode;
expect(mode & ~fs.constants.S_IFREG).to.be(parseInt('0755', 8));
});
});
describe('backupBoxAndApps', function () {
let backupInfo1;
const backupConfig = {
provider: 'filesystem',
backupFolder: path.join(os.tmpdir(), 'backupstask-test-filesystem'),
format: 'tgz',
retentionPolicy: { keepWithinSecs: 10000 },
schedulePattern: '00 00 23 * * *'
};
before(function (done) {
fs.rmSync(backupConfig.backupFolder, { recursive: true, force: true });
settings.setBackupConfig(backupConfig, done);
});
after(function () {
fs.rmSync(backupConfig.backupFolder, { recursive: true, force: true });
});
function createBackup(callback) {
backups.startBackupTask({ username: 'test' }, async function (error, taskId) { // this call does not wait for the backup!
if (error) return callback(error);
// eslint-disable-next-line no-constant-condition
while (true) {
await delay(1000);
const p = await tasks.get(taskId);
if (p.percent !== 100) continue;
if (p.error) return callback(new Error(`backup failed: ${p.error.message}`));
if (!p.result) return callback(new Error('backup has no result:' + p));
const [error, result] = await safe(backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, 1, 1));
if (error) return callback(error);
if (result.length !== 1) return callback(new Error('result is not of length 1'));
// the task progress and the db entry is set in the worker. wait for 2 seconds for backup lock to get released in parent process
await delay(2000);
callback(null, result[0]);
}
});
}
it('can backup', function (done) {
// arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/
if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) {
console.log('test skipped because of MariaDB');
return done();
}
createBackup(function (error, result) {
expect(error).to.be(null);
expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2);
backupInfo1 = result;
done();
});
});
it('can take another backup', function (done) {
// arch only has maria db which lacks some mysqldump options we need, this is only here to allow running the tests :-/
if (require('child_process').execSync('/usr/bin/mysqldump --version').toString().indexOf('MariaDB') !== -1) {
console.log('test skipped because of MariaDB');
return done();
}
createBackup(function (error, result) {
expect(error).to.be(null);
expect(fs.statSync(path.join(backupConfig.backupFolder, 'snapshot/box.tar.gz')).nlink).to.be(2); // hard linked to a rotated backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${result.id}.tar.gz`)).nlink).to.be(2); // hard linked to new backup
expect(fs.statSync(path.join(backupConfig.backupFolder, `${backupInfo1.id}.tar.gz`)).nlink).to.be(1); // not hard linked anymore
done();
});
});
});
});

2
src/test/common.js
View File

@@ -17,6 +17,7 @@ const appdb = require('../appdb.js'),
rimraf = require('rimraf'),
settings = require('../settings.js'),
settingsdb = require('../settingsdb.js'),
tasks = require('../tasks.js'),
userdb = require('../userdb.js'),
users = require('../users.js');
@@ -182,6 +183,7 @@ function setup(done) {
(done) => mailboxdb.addMailbox(exports.MAILBOX_NAME, DOMAIN.domain, { ownerId: USER.id, ownerType: mail.OWNERTYPE_USER, active: true }, done),
(done) => mailboxdb.setAliasesForName(exports.MAILBOX_NAME, DOMAIN.domain, [ { name: exports.ALIAS_NAME, domain: DOMAIN.domain} ], done),
tasks.stopAllTasks,
], done);
}

38
src/test/storage-test.js
View File

@@ -5,7 +5,8 @@
'use strict';
var BoxError = require('../boxerror.js'),
const BoxError = require('../boxerror.js'),
common = require('./common.js'),
execSync = require('child_process').execSync,
expect = require('expect.js'),
filesystem = require('../storage/filesystem.js'),
@@ -17,19 +18,26 @@ var BoxError = require('../boxerror.js'),
rimraf = require('rimraf'),
recursive_readdir = require('recursive-readdir'),
s3 = require('../storage/s3.js'),
settings = require('../settings.js'),
gcs = require('../storage/gcs.js'),
chunk = require('lodash.chunk');
describe('Storage', function () {
const { setup, cleanup } = common;
before(setup);
after(cleanup);
describe('filesystem', function () {
let gTmpFolder;
var gTmpFolder;
var gBackupConfig = {
const gBackupConfig = {
provider: 'filesystem',
key: 'key',
backupFolder: null,
format: 'tgz'
format: 'tgz',
retentionPolicy: { keepWithinSecs: 10000 },
schedulePattern: '00 00 23 * * *'
};
before(function (done) {
@@ -45,6 +53,26 @@ describe('Storage', function () {
done();
});
it('fails to set backup config for bad folder', function (done) {
const tmp = Object.assign({}, gBackupConfig, { backupFolder: '/root/oof' });
settings.setBackupConfig(tmp, function (error) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.BAD_FIELD);
done();
});
});
it('succeeds to set backup config', function (done) {
settings.setBackupConfig(gBackupConfig, function (error) {
expect(error).to.be(null);
expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created
done();
});
});
it('can upload', function (done) {
var sourceFile = path.join(__dirname, 'storage/data/test.txt');
var sourceStream = fs.createReadStream(sourceFile);