jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: fix cleanup

Browse Source 
The various changes are:
* Latest backup is always kept for box and app backups
* If the latest backup is part of the policy, it is not counted twice
* Latest backup comes into action only when all backups are outside the retention policy
* For uninstalled apps, latest backup is not preserved
* This way the latest backup of apps that are not referenced in box backup is preserved.
  (for example, for stopped apps)

fixes #692
This commit is contained in:
Girish Ramakrishnan
2020-06-14 15:09:04 -07:00
parent 2601d2945d
commit 129cbb5beb
3 changed files with 135 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/test/backups-test.js
+90 -39
View File
@@ -6,19 +6,22 @@
'use strict';
var async = require('async'),
var appdb = require('../appdb.js'),
async = require('async'),
backupdb = require('../backupdb.js'),
backups = require('../backups.js'),
BoxError = require('../boxerror.js'),
createTree = require('./common.js').createTree,
database = require('../database'),
DataLayout = require('../datalayout.js'),
domains = require('../domains.js'),
expect = require('expect.js'),
fs = require('fs'),
os = require('os'),
moment = require('moment'),
path = require('path'),
rimraf = require('rimraf'),
settingsdb = require('../settingsdb.js'),
settings = require('../settings.js'),
tasks = require('../tasks.js');
@@ -69,73 +72,107 @@ function cleanupBackups(callback) {
}
describe('retention policy', function () {
it('always keeps if reason is set', function () {
let backup = { keepReason: 'somereason' };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1 });
expect(backup.keepReason).to.be('somereason');
it('keeps latest', function () {
let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []);
expect(backup.keepReason).to.be('latest');
});
it('does not keep latest', function () {
let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []);
expect(backup.keepReason).to.be(undefined);
});
it('always keeps forever policy', function () {
let backup = { creationTime: new Date() };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1 });
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []);
expect(backup.keepReason).to.be('keepWithinSecs');
});
it('preserveSecs takes precedence', function () {
let backup = { creationTime: new Date(), preserveSecs: 3000 };
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1 });
backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []);
expect(backup.keepReason).to.be('preserveSecs');
});
it('1 daily', function () {
let b = [
{ id: '1', creationTime: moment().toDate() },
{ id: '2', creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', creationTime: moment().subtract(20, 'h').toDate() },
{ id: '4', creationTime: moment().subtract(5, 'd').toDate() }
{ id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
];
backups._applyBackupRetentionPolicy(b, { keepDaily: 1 });
backups._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []);
expect(b[0].keepReason).to.be('keepDaily');
expect(b[1].keepReason).to.be(undefined);
expect(b[2].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be(undefined);
});
it('2 daily, 1 weekly', function () {
let b = [
{ id: '1', creationTime: moment().toDate() },
{ id: '2', creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', creationTime: moment().subtract(26, 'h').toDate() },
{ id: '4', creationTime: moment().subtract(5, 'd').toDate() }
{ id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() },
{ id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() },
{ id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() }
];
backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1 });
backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []);
expect(b[0].keepReason).to.be('keepDaily'); // today
expect(b[1].keepReason).to.be('keepWeekly');
expect(b[2].keepReason).to.be('keepDaily'); // yesterday
expect(b[3].keepReason).to.be(undefined);
expect(b[1].keepReason).to.be('keepWeekly'); // today
expect(b[2].keepReason).to.be(undefined);
expect(b[3].keepReason).to.be('keepDaily'); // yesterday
expect(b[4].discardReason).to.be('error'); // errored
expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long
expect(b[6].keepReason).to.be(undefined); // outside retention policy
});
it('2 daily, 3 monthly, 1 yearly', function () {
let b = [
{ id: '1', creationTime: moment().toDate() },
{ id: '2', creationTime: moment().subtract(3, 'h').toDate() },
{ id: '3', creationTime: moment().subtract(26, 'h').toDate() },
{ id: '4', creationTime: moment().subtract(32, 'd').toDate() },
{ id: '5', creationTime: moment().subtract(63, 'd').toDate() },
{ id: '6', creationTime: moment().subtract(65, 'd').toDate() },
{ id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() },
{ id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() },
{ id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() },
{ id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() },
{ id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() },
{ id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() },
{ id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() },
{ id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() },
{ id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() },
{ id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() },
];
backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1 });
expect(b[0].keepReason).to.be('keepDaily'); // today
expect(b[1].keepReason).to.be('keepMonthly');
expect(b[2].keepReason).to.be('keepDaily'); // yesterday
backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []);
expect(b[0].keepReason).to.be('creating');
expect(b[1].discardReason).to.be('error'); // errored
expect(b[2].keepReason).to.be('keepDaily');
expect(b[3].keepReason).to.be('keepMonthly');
expect(b[4].keepReason).to.be('keepMonthly');
expect(b[5].keepReason).to.be('keepYearly');
expect(b[4].keepReason).to.be('keepYearly');
expect(b[5].keepReason).to.be('keepDaily'); // yesterday
expect(b[6].discardReason).to.be('creating-too-long'); // errored
expect(b[7].keepReason).to.be('keepMonthly');
expect(b[8].keepReason).to.be('keepMonthly');
expect(b[9].keepReason).to.be(undefined);
});
});
describe('backups', function () {
const DOMAIN_0 = {
domain: 'example.com',
zoneName: 'example.com',
provider: 'manual',
config: { },
fallbackCertificate: null,
tlsConfig: { provider: 'fallback' }
};
const AUDIT_SOURCE = { ip: '1.2.3.4' };
const manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
before(function (done) {
const BACKUP_DIR = path.join(os.tmpdir(), 'cloudron-backup-test');
@@ -143,13 +180,15 @@ describe('backups', function () {
fs.mkdir.bind(null, BACKUP_DIR, { recursive: true }),
database.initialize,
database._clear,
settings.setBackupConfig.bind(null, {
settings.setAdmin.bind(null, DOMAIN_0.domain, 'my.' + DOMAIN_0.domain),
domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0, AUDIT_SOURCE),
settingsdb.set.bind(null, settings.BACKUP_CONFIG_KEY, JSON.stringify({
provider: 'filesystem',
password: 'supersecret',
backupFolder: BACKUP_DIR,
retentionPolicy: { keepWithinSecs: 1 },
format: 'tgz'
})
}))
], done);
});
@@ -172,7 +211,7 @@ describe('backups', function () {
format: 'tgz'
};
var BACKUP_0_APP_0 = {
var BACKUP_0_APP_0 = { // backup of installed app
id: 'backup-app-00',
identifier: 'app0',
encryptionVersion: null,
@@ -184,7 +223,7 @@ describe('backups', function () {
format: 'tgz'
};
var BACKUP_0_APP_1 = {
var BACKUP_0_APP_1 = { // this app is uninstalled
id: 'backup-app-01',
identifier: 'app1',
encryptionVersion: null,
@@ -232,6 +271,14 @@ describe('backups', function () {
format: 'tgz'
};
before(function (done) {
appdb.add('app0', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { installationState: 'installed', runState: 'running', mailboxDomain: DOMAIN_0.domain }, done);
});
after(function (done) {
appdb.del('app0', done);
});
it('succeeds without backups', function (done) {
cleanupBackups(done);
});
@@ -253,7 +300,7 @@ describe('backups', function () {
expect(result.length).to.equal(1);
expect(result[0].id).to.equal(BACKUP_1.id);
// check that app backups are gone as well
// check that app backups are gone as well. only backup_1 will remain
backupdb.get(BACKUP_0_APP_0.id, function (error) {
expect(error).to.be.a(BoxError);
expect(error.reason).to.equal(BoxError.NOT_FOUND);
@@ -274,7 +321,7 @@ describe('backups', function () {
expect(result.length).to.equal(1);
expect(result[0].id).to.equal(BACKUP_1.id);
// check that app backups are also still there
// check that app backups are also still there. backup_1 is still there
backupdb.get(BACKUP_1_APP_0.id, function (error, result) {
expect(error).to.not.be.ok();
expect(result.id).to.equal(BACKUP_1_APP_0.id);
@@ -286,6 +333,7 @@ describe('backups', function () {
});
it('succeeds for app backups not referenced by a box backup', function (done) {
// add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there
async.eachSeries([BACKUP_0_APP_0, BACKUP_0_APP_1], (b, done) => backupdb.add(b.id, b, done), function (error) {
expect(error).to.not.be.ok();
@@ -296,7 +344,10 @@ describe('backups', function () {
backupdb.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000, function (error, result) {
expect(error).to.not.be.ok();
expect(result.length).to.equal(2);
expect(result.length).to.equal(3);
expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved
expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box
expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box
done();
});