jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: fix cleanup

Browse Source 
The various changes are:
* Latest backup is always kept for box and app backups
* If the latest backup is part of the policy, it is not counted twice
* Latest backup comes into action only when all backups are outside the retention policy
* For uninstalled apps, latest backup is not preserved
* This way the latest backup of apps that are not referenced in box backup is preserved.
  (for example, for stopped apps)

fixes #692
This commit is contained in:
Girish Ramakrishnan
2020-06-14 15:09:04 -07:00
parent 2601d2945d
commit 129cbb5beb
3 changed files with 135 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backups.js
+44 -39
View File
@@ -78,7 +78,8 @@ var addons = require('./addons.js'),
tasks = require('./tasks.js'),
TransformStream = require('stream').Transform,
util = require('util'),
zlib = require('zlib');
zlib = require('zlib'),
_ = require('underscore');
const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
const COLLECTD_CONFIG_EJS = fs.readFileSync(__dirname + '/collectd/cloudron-backup.ejs', { encoding: 'utf8' });
@@ -1249,16 +1250,22 @@ function ensureBackup(auditSource, callback) {
}
// backups must be descending in creationTime
function applyBackupRetentionPolicy(backups, policy) {
function applyBackupRetentionPolicy(backups, policy, referencedBackupIds) {
assert(Array.isArray(backups));
assert.strictEqual(typeof policy, 'object');
assert(Array.isArray(referencedBackupIds));
const now = new Date();
for (const backup of backups) {
if (backup.keepReason) continue; // already kept for some other reason
if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
if (backup.state === exports.BACKUP_STATE_ERROR) {
backup.discardReason = 'error';
} else if (backup.state === exports.BACKUP_STATE_CREATING) {
if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
else backup.discardReason = 'creating-too-long';
} else if (referencedBackupIds.includes(backup.id)) {
backup.keepReason = 'reference';
} else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
backup.keepReason = 'preserveSecs';
} else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
backup.keepReason = 'keepWithinSecs';
@@ -1280,7 +1287,7 @@ function applyBackupRetentionPolicy(backups, policy) {
let lastPeriod = null, keptSoFar = 0;
for (const backup of backups) {
if (backup.keepReason) continue; // already kept for some other reason
if (backup.discardReason || backup.keepReason) continue; // already kept or discarded for some reason
const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
if (period === lastPeriod) continue; // already kept for this period
@@ -1290,8 +1297,13 @@ function applyBackupRetentionPolicy(backups, policy) {
}
}
if (policy.keepLatest) {
let latestNormalBackup = backups.find(b => b.state === exports.BACKUP_STATE_NORMAL);
if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
}
for (const backup of backups) {
if (backup.keepReason) debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason}`);
debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
}
}
@@ -1340,27 +1352,37 @@ function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallbac
let removedAppBackupIds = [];
// we clean app backups of any state because the ones to keep are determined by the box cleanup code
backupdb.getByTypePaged(exports.BACKUP_TYPE_APP, 1, 1000, function (error, appBackups) {
apps.getAll(function (error, allApps) {
if (error) return callback(error);
for (const appBackup of appBackups) { // set the reason so that policy filter can skip it
if (referencedAppBackupIds.includes(appBackup.id)) appBackup.keepReason = 'reference';
}
const allAppIds = allApps.map(a => a.id);
applyBackupRetentionPolicy(appBackups, backupConfig.retentionPolicy);
backupdb.getByTypePaged(exports.BACKUP_TYPE_APP, 1, 1000, function (error, appBackups) {
if (error) return callback(error);
async.eachSeries(appBackups, function iterator(appBackup, iteratorDone) {
if (appBackup.keepReason) return iteratorDone();
// collate the backups by app id. note that the app could already have been uninstalled
let appBackupsById = {};
for (const appBackup of appBackups) {
if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
appBackupsById[appBackup.identifier].push(appBackup);
}
progressCallback({ message: `Removing app backup ${appBackup.id}`});
// apply backup policy per app. keep latest backup only for existing apps
let appBackupsToRemove = [];
for (const appId of Object.keys(appBackupsById)) {
applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
}
removedAppBackupIds.push(appBackup.id);
cleanupBackup(backupConfig, appBackup, progressCallback, iteratorDone);
}, function () {
debug('cleanupAppBackups: done');
async.eachSeries(appBackupsToRemove, function iterator(appBackup, iteratorDone) {
progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
removedAppBackupIds.push(appBackup.id);
cleanupBackup(backupConfig, appBackup, progressCallback, iteratorDone);
}, function () {
debug('cleanupAppBackups: done');
callback(null, removedAppBackupIds);
callback(null, removedAppBackupIds);
});
});
});
}
@@ -1375,24 +1397,7 @@ function cleanupBoxBackups(backupConfig, progressCallback, callback) {
backupdb.getByTypePaged(exports.BACKUP_TYPE_BOX, 1, 1000, function (error, boxBackups) {
if (error) return callback(error);
if (boxBackups.length === 0) return callback(null, { removedBoxBackupIds, referencedAppBackupIds });
// search for the first valid backup
var i;
for (i = 0; i < boxBackups.length; i++) {
if (boxBackups[i].state === exports.BACKUP_STATE_NORMAL) break;
}
// keep the first valid backup
if (i !== boxBackups.length) {
debug('cleanupBoxBackups: preserving box backup %s (%j)', boxBackups[i].id, boxBackups[i].dependsOn);
referencedAppBackupIds = boxBackups[i].dependsOn;
boxBackups.splice(i, 1);
} else {
debug('cleanupBoxBackups: no box backup to preserve');
}
applyBackupRetentionPolicy(boxBackups, backupConfig.retentionPolicy);
applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
async.eachSeries(boxBackups, function iterator(boxBackup, iteratorNext) {
if (boxBackup.keepReason) {