diff --git a/CHANGES b/CHANGES index 572546923..cb6e74195 100644 --- a/CHANGES +++ b/CHANGES @@ -1972,4 +1972,5 @@ * add forumUrl to app manifest * postgresql: add unaccent extension for peertube * mail: Add Auto-Submitted header to NDRs +* backups: ensure that the latest backup of installed apps is always preserved diff --git a/src/backups.js b/src/backups.js index 1e137ae4d..ffae90d13 100644 --- a/src/backups.js +++ b/src/backups.js @@ -78,7 +78,8 @@ var addons = require('./addons.js'), tasks = require('./tasks.js'), TransformStream = require('stream').Transform, util = require('util'), - zlib = require('zlib'); + zlib = require('zlib'), + _ = require('underscore'); const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js'); const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' }); @@ -1249,16 +1250,22 @@ function ensureBackup(auditSource, callback) { } // backups must be descending in creationTime -function applyBackupRetentionPolicy(backups, policy) { +function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) { assert(Array.isArray(backups)); assert.strictEqual(typeof policy, 'object'); + assert(Array.isArray(referencedBackupIds)); const now = new Date(); for (const backup of backups) { - if (backup.keepReason) continue; // already kept for some other reason - - if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) { + if (backup.state === exports.BACKUP_STATE_ERROR) { + backup.discardReason = 'error'; + } else if (backup.state === exports.BACKUP_STATE_CREATING) { + if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating'; + else backup.discardReason = 'creating-too-long'; + } else if (referencedBackupIds.includes(backup.id)) { + backup.keepReason = 'reference'; + } else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) { backup.keepReason = 'preserveSecs'; } else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) { backup.keepReason = 'keepWithinSecs'; @@ -1280,7 +1287,7 @@ function applyBackupRetentionPolicy(backups, policy) { let lastPeriod = null, keptSoFar = 0; for (const backup of backups) { - if (backup.keepReason) continue; // already kept for some other reason + if (backup.discardReason || backup.keepReason) continue; // already kept or discarded for some reason const period = moment(backup.creationTime).format(KEEP_FORMATS[format]); if (period === lastPeriod) continue; // already kept for this period @@ -1290,8 +1297,13 @@ function applyBackupRetentionPolicy(backups, policy) { } } + if (policy.keepLatest) { + let latestNormalBackup = backups.find(b => b.state === exports.BACKUP_STATE_NORMAL); + if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest'; + } + for (const backup of backups) { - if (backup.keepReason) debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason}`); + debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`); } } @@ -1340,27 +1352,37 @@ function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallbac let removedAppBackupIds = []; - // we clean app backups of any state because the ones to keep are determined by the box cleanup code - backupdb.getByTypePaged(exports.BACKUP_TYPE_APP, 1, 1000, function (error, appBackups) { + apps.getAll(function (error, allApps) { if (error) return callback(error); - for (const appBackup of appBackups) { // set the reason so that policy filter can skip it - if (referencedAppBackupIds.includes(appBackup.id)) appBackup.keepReason = 'reference'; - } + const allAppIds = allApps.map(a => a.id); - applyBackupRetentionPolicy(appBackups, backupConfig.retentionPolicy); + backupdb.getByTypePaged(exports.BACKUP_TYPE_APP, 1, 1000, function (error, appBackups) { + if (error) return callback(error); - async.eachSeries(appBackups, function iterator(appBackup, iteratorDone) { - if (appBackup.keepReason) return iteratorDone(); + // collate the backups by app id. note that the app could already have been uninstalled + let appBackupsById = {}; + for (const appBackup of appBackups) { + if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = []; + appBackupsById[appBackup.identifier].push(appBackup); + } - progressCallback({ message: `Removing app backup ${appBackup.id}`}); + // apply backup policy per app. keep latest backup only for existing apps + let appBackupsToRemove = []; + for (const appId of Object.keys(appBackupsById)) { + applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds); + appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason)); + } - removedAppBackupIds.push(appBackup.id); - cleanupBackup(backupConfig, appBackup, progressCallback, iteratorDone); - }, function () { - debug('cleanupAppBackups: done'); + async.eachSeries(appBackupsToRemove, function iterator(appBackup, iteratorDone) { + progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`}); + removedAppBackupIds.push(appBackup.id); + cleanupBackup(backupConfig, appBackup, progressCallback, iteratorDone); + }, function () { + debug('cleanupAppBackups: done'); - callback(null, removedAppBackupIds); + callback(null, removedAppBackupIds); + }); }); }); } @@ -1375,24 +1397,7 @@ function cleanupBoxBackups(backupConfig, progressCallback, callback) { backupdb.getByTypePaged(exports.BACKUP_TYPE_BOX, 1, 1000, function (error, boxBackups) { if (error) return callback(error); - if (boxBackups.length === 0) return callback(null, { removedBoxBackupIds, referencedAppBackupIds }); - - // search for the first valid backup - var i; - for (i = 0; i < boxBackups.length; i++) { - if (boxBackups[i].state === exports.BACKUP_STATE_NORMAL) break; - } - - // keep the first valid backup - if (i !== boxBackups.length) { - debug('cleanupBoxBackups: preserving box backup %s (%j)', boxBackups[i].id, boxBackups[i].dependsOn); - referencedAppBackupIds = boxBackups[i].dependsOn; - boxBackups.splice(i, 1); - } else { - debug('cleanupBoxBackups: no box backup to preserve'); - } - - applyBackupRetentionPolicy(boxBackups, backupConfig.retentionPolicy); + applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */); async.eachSeries(boxBackups, function iterator(boxBackup, iteratorNext) { if (boxBackup.keepReason) { diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 842c380b7..e87cf5c3d 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -6,19 +6,22 @@ 'use strict'; -var async = require('async'), +var appdb = require('../appdb.js'), + async = require('async'), backupdb = require('../backupdb.js'), backups = require('../backups.js'), BoxError = require('../boxerror.js'), createTree = require('./common.js').createTree, database = require('../database'), DataLayout = require('../datalayout.js'), + domains = require('../domains.js'), expect = require('expect.js'), fs = require('fs'), os = require('os'), moment = require('moment'), path = require('path'), rimraf = require('rimraf'), + settingsdb = require('../settingsdb.js'), settings = require('../settings.js'), tasks = require('../tasks.js'); @@ -69,73 +72,107 @@ function cleanupBackups(callback) { } describe('retention policy', function () { - it('always keeps if reason is set', function () { - let backup = { keepReason: 'somereason' }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1 }); - expect(backup.keepReason).to.be('somereason'); + it('keeps latest', function () { + let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; + backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); + expect(backup.keepReason).to.be('latest'); + }); + + it('does not keep latest', function () { + let backup = { creationTime: moment().subtract(5, 's').toDate(), state: backups.BACKUP_STATE_NORMAL }; + backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: false }, []); + expect(backup.keepReason).to.be(undefined); }); it('always keeps forever policy', function () { let backup = { creationTime: new Date() }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1 }); + backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: -1, keepLatest: true }, []); expect(backup.keepReason).to.be('keepWithinSecs'); }); it('preserveSecs takes precedence', function () { let backup = { creationTime: new Date(), preserveSecs: 3000 }; - backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1 }); + backups._applyBackupRetentionPolicy([backup], { keepWithinSecs: 1, keepLatest: true }, []); expect(backup.keepReason).to.be('preserveSecs'); }); it('1 daily', function () { let b = [ - { id: '1', creationTime: moment().toDate() }, - { id: '2', creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', creationTime: moment().subtract(20, 'h').toDate() }, - { id: '4', creationTime: moment().subtract(5, 'd').toDate() } + { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(20, 'h').toDate() }, + { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; - backups._applyBackupRetentionPolicy(b, { keepDaily: 1 }); + backups._applyBackupRetentionPolicy(b, { keepDaily: 1, keepLatest: true }, []); expect(b[0].keepReason).to.be('keepDaily'); expect(b[1].keepReason).to.be(undefined); expect(b[2].keepReason).to.be(undefined); expect(b[3].keepReason).to.be(undefined); + expect(b[3].keepReason).to.be(undefined); }); it('2 daily, 1 weekly', function () { let b = [ - { id: '1', creationTime: moment().toDate() }, - { id: '2', creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', creationTime: moment().subtract(26, 'h').toDate() }, - { id: '4', creationTime: moment().subtract(5, 'd').toDate() } + { id: '0', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '1', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, + { id: '4', state: backups.BACKUP_STATE_ERROR, creationTime: moment().subtract(32, 'h').toDate() }, + { id: '5', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(50, 'h').toDate() }, + { id: '6', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(5, 'd').toDate() } ]; - backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1 }); + backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepWeekly: 1, keepLatest: false }, []); expect(b[0].keepReason).to.be('keepDaily'); // today - expect(b[1].keepReason).to.be('keepWeekly'); - expect(b[2].keepReason).to.be('keepDaily'); // yesterday - expect(b[3].keepReason).to.be(undefined); + expect(b[1].keepReason).to.be('keepWeekly'); // today + expect(b[2].keepReason).to.be(undefined); + expect(b[3].keepReason).to.be('keepDaily'); // yesterday + expect(b[4].discardReason).to.be('error'); // errored + expect(b[5].discardReason).to.be('creating-too-long'); // creating for too long + expect(b[6].keepReason).to.be(undefined); // outside retention policy }); it('2 daily, 3 monthly, 1 yearly', function () { let b = [ - { id: '1', creationTime: moment().toDate() }, - { id: '2', creationTime: moment().subtract(3, 'h').toDate() }, - { id: '3', creationTime: moment().subtract(26, 'h').toDate() }, - { id: '4', creationTime: moment().subtract(32, 'd').toDate() }, - { id: '5', creationTime: moment().subtract(63, 'd').toDate() }, - { id: '6', creationTime: moment().subtract(65, 'd').toDate() }, + { id: '0', state: backups.BACKUP_STATE_CREATING, creationTime: moment().toDate() }, + { id: '1', state: backups.BACKUP_STATE_ERROR, creationTime: moment().toDate() }, + { id: '2', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().toDate() }, + { id: '3', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(1, 'h').toDate() }, + { id: '4', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(3, 'h').toDate() }, + { id: '5', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(26, 'h').toDate() }, + { id: '6', state: backups.BACKUP_STATE_CREATING, creationTime: moment().subtract(49, 'h').toDate() }, + { id: '7', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(51, 'd').toDate() }, + { id: '8', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(84, 'd').toDate() }, + { id: '9', state: backups.BACKUP_STATE_NORMAL, creationTime: moment().subtract(97, 'd').toDate() }, ]; - backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1 }); - expect(b[0].keepReason).to.be('keepDaily'); // today - expect(b[1].keepReason).to.be('keepMonthly'); - expect(b[2].keepReason).to.be('keepDaily'); // yesterday + backups._applyBackupRetentionPolicy(b, { keepDaily: 2, keepMonthly: 3, keepYearly: 1, keepLatest: true }, []); + expect(b[0].keepReason).to.be('creating'); + expect(b[1].discardReason).to.be('error'); // errored + expect(b[2].keepReason).to.be('keepDaily'); expect(b[3].keepReason).to.be('keepMonthly'); - expect(b[4].keepReason).to.be('keepMonthly'); - expect(b[5].keepReason).to.be('keepYearly'); + expect(b[4].keepReason).to.be('keepYearly'); + expect(b[5].keepReason).to.be('keepDaily'); // yesterday + expect(b[6].discardReason).to.be('creating-too-long'); // errored + expect(b[7].keepReason).to.be('keepMonthly'); + expect(b[8].keepReason).to.be('keepMonthly'); + expect(b[9].keepReason).to.be(undefined); }); }); describe('backups', function () { + const DOMAIN_0 = { + domain: 'example.com', + zoneName: 'example.com', + provider: 'manual', + config: { }, + fallbackCertificate: null, + tlsConfig: { provider: 'fallback' } + }; + const AUDIT_SOURCE = { ip: '1.2.3.4' }; + + const manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } }; + before(function (done) { const BACKUP_DIR = path.join(os.tmpdir(), 'cloudron-backup-test'); @@ -143,13 +180,15 @@ describe('backups', function () { fs.mkdir.bind(null, BACKUP_DIR, { recursive: true }), database.initialize, database._clear, - settings.setBackupConfig.bind(null, { + settings.setAdmin.bind(null, DOMAIN_0.domain, 'my.' + DOMAIN_0.domain), + domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0, AUDIT_SOURCE), + settingsdb.set.bind(null, settings.BACKUP_CONFIG_KEY, JSON.stringify({ provider: 'filesystem', password: 'supersecret', backupFolder: BACKUP_DIR, retentionPolicy: { keepWithinSecs: 1 }, format: 'tgz' - }) + })) ], done); }); @@ -172,7 +211,7 @@ describe('backups', function () { format: 'tgz' }; - var BACKUP_0_APP_0 = { + var BACKUP_0_APP_0 = { // backup of installed app id: 'backup-app-00', identifier: 'app0', encryptionVersion: null, @@ -184,7 +223,7 @@ describe('backups', function () { format: 'tgz' }; - var BACKUP_0_APP_1 = { + var BACKUP_0_APP_1 = { // this app is uninstalled id: 'backup-app-01', identifier: 'app1', encryptionVersion: null, @@ -232,6 +271,14 @@ describe('backups', function () { format: 'tgz' }; + before(function (done) { + appdb.add('app0', 'appStoreId', manifest, 'location', DOMAIN_0.domain, [ ] /* portBindings */, { installationState: 'installed', runState: 'running', mailboxDomain: DOMAIN_0.domain }, done); + }); + + after(function (done) { + appdb.del('app0', done); + }); + it('succeeds without backups', function (done) { cleanupBackups(done); }); @@ -253,7 +300,7 @@ describe('backups', function () { expect(result.length).to.equal(1); expect(result[0].id).to.equal(BACKUP_1.id); - // check that app backups are gone as well + // check that app backups are gone as well. only backup_1 will remain backupdb.get(BACKUP_0_APP_0.id, function (error) { expect(error).to.be.a(BoxError); expect(error.reason).to.equal(BoxError.NOT_FOUND); @@ -274,7 +321,7 @@ describe('backups', function () { expect(result.length).to.equal(1); expect(result[0].id).to.equal(BACKUP_1.id); - // check that app backups are also still there + // check that app backups are also still there. backup_1 is still there backupdb.get(BACKUP_1_APP_0.id, function (error, result) { expect(error).to.not.be.ok(); expect(result.id).to.equal(BACKUP_1_APP_0.id); @@ -286,6 +333,7 @@ describe('backups', function () { }); it('succeeds for app backups not referenced by a box backup', function (done) { + // add two dangling app backups not referenced by box backup. app1 is uninstalled. app0 is there async.eachSeries([BACKUP_0_APP_0, BACKUP_0_APP_1], (b, done) => backupdb.add(b.id, b, done), function (error) { expect(error).to.not.be.ok(); @@ -296,7 +344,10 @@ describe('backups', function () { backupdb.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000, function (error, result) { expect(error).to.not.be.ok(); - expect(result.length).to.equal(2); + expect(result.length).to.equal(3); + expect(result[0].id).to.be(BACKUP_0_APP_0.id); // because app is installed, latest backup is preserved + expect(result[1].id).to.be(BACKUP_1_APP_0.id); // referenced by box + expect(result[2].id).to.be(BACKUP_1_APP_1.id); // referenced by box done(); });