300 lines
12 KiB
JavaScript
300 lines
12 KiB
JavaScript
'use strict';
|
|
|
|
exports = module.exports = {
|
|
run,
|
|
|
|
_applyBackupRetentionPolicy: applyBackupRetentionPolicy
|
|
};
|
|
|
|
const apps = require('./apps.js'),
|
|
assert = require('assert'),
|
|
async = require('async'),
|
|
backups = require('./backups.js'),
|
|
BoxError = require('./boxerror.js'),
|
|
constants = require('./constants.js'),
|
|
debug = require('debug')('box:backupcleaner'),
|
|
moment = require('moment'),
|
|
path = require('path'),
|
|
paths = require('./paths.js'),
|
|
safe = require('safetydance'),
|
|
settings = require('./settings.js'),
|
|
storage = require('./storage.js'),
|
|
util = require('util'),
|
|
_ = require('underscore');
|
|
|
|
function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
|
|
assert(Array.isArray(allBackups));
|
|
assert.strictEqual(typeof policy, 'object');
|
|
assert(Array.isArray(referencedBackupIds));
|
|
|
|
const now = new Date();
|
|
|
|
for (const backup of allBackups) {
|
|
if (backup.state === backups.BACKUP_STATE_ERROR) {
|
|
backup.discardReason = 'error';
|
|
} else if (backup.state === backups.BACKUP_STATE_CREATING) {
|
|
if ((now - backup.creationTime) < 48*60*60*1000) backup.keepReason = 'creating';
|
|
else backup.discardReason = 'creating-too-long';
|
|
} else if (referencedBackupIds.includes(backup.id)) {
|
|
backup.keepReason = 'reference';
|
|
} else if ((now - backup.creationTime) < (backup.preserveSecs * 1000)) {
|
|
backup.keepReason = 'preserveSecs';
|
|
} else if ((now - backup.creationTime < policy.keepWithinSecs * 1000) || policy.keepWithinSecs < 0) {
|
|
backup.keepReason = 'keepWithinSecs';
|
|
}
|
|
}
|
|
|
|
const KEEP_FORMATS = {
|
|
keepDaily: 'Y-M-D',
|
|
keepWeekly: 'Y-W',
|
|
keepMonthly: 'Y-M',
|
|
keepYearly: 'Y'
|
|
};
|
|
|
|
for (const format of [ 'keepDaily', 'keepWeekly', 'keepMonthly', 'keepYearly' ]) {
|
|
if (!(format in policy)) continue;
|
|
|
|
const n = policy[format]; // we want to keep "n" backups of format
|
|
if (!n) continue; // disabled rule
|
|
|
|
let lastPeriod = null, keptSoFar = 0;
|
|
for (const backup of allBackups) {
|
|
if (backup.discardReason) continue; // already discarded for some reason
|
|
if (backup.keepReason && backup.keepReason !== 'reference') continue; // kept for some other reason
|
|
const period = moment(backup.creationTime).format(KEEP_FORMATS[format]);
|
|
if (period === lastPeriod) continue; // already kept for this period
|
|
|
|
lastPeriod = period;
|
|
backup.keepReason = backup.keepReason ? `${backup.keepReason}+${format}` : format;
|
|
if (++keptSoFar === n) break;
|
|
}
|
|
}
|
|
|
|
if (policy.keepLatest) {
|
|
let latestNormalBackup = allBackups.find(b => b.state === backups.BACKUP_STATE_NORMAL);
|
|
if (latestNormalBackup && !latestNormalBackup.keepReason) latestNormalBackup.keepReason = 'latest';
|
|
}
|
|
|
|
for (const backup of allBackups) {
|
|
debug(`applyBackupRetentionPolicy: ${backup.id} ${backup.type} ${backup.keepReason || backup.discardReason || 'unprocessed'}`);
|
|
}
|
|
}
|
|
|
|
async function cleanupBackup(backupConfig, backup, progressCallback) {
|
|
assert.strictEqual(typeof backupConfig, 'object');
|
|
assert.strictEqual(typeof backup, 'object');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
|
|
|
|
return new Promise((resolve) => {
|
|
function done(error) {
|
|
if (error) {
|
|
debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
|
|
return resolve();
|
|
}
|
|
|
|
// prune empty directory if possible
|
|
storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
|
|
if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
|
|
|
|
const [delError] = await safe(backups.del(backup.id));
|
|
if (delError) debug('cleanupBackup: error removing from database', delError);
|
|
else debug('cleanupBackup: removed %s', backup.id);
|
|
|
|
resolve();
|
|
});
|
|
}
|
|
|
|
if (backup.format ==='tgz') {
|
|
progressCallback({ message: `${backup.id}: Removing ${backupFilePath}`});
|
|
storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
|
|
} else {
|
|
const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
|
|
events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
|
|
events.on('done', done);
|
|
}
|
|
});
|
|
}
|
|
|
|
function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, callback) {
|
|
assert.strictEqual(typeof backupConfig, 'object');
|
|
assert(Array.isArray(referencedAppBackupIds));
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
let removedAppBackupIds = [];
|
|
|
|
apps.getAll(async function (error, allApps) {
|
|
if (error) return callback(error);
|
|
|
|
const allAppIds = allApps.map(a => a.id);
|
|
|
|
const [getError, appBackups] = await safe(backups.getByTypePaged(backups.BACKUP_TYPE_APP, 1, 1000));
|
|
if (getError) return callback(getError);
|
|
|
|
// collate the backups by app id. note that the app could already have been uninstalled
|
|
let appBackupsById = {};
|
|
for (const appBackup of appBackups) {
|
|
if (!appBackupsById[appBackup.identifier]) appBackupsById[appBackup.identifier] = [];
|
|
appBackupsById[appBackup.identifier].push(appBackup);
|
|
}
|
|
|
|
// apply backup policy per app. keep latest backup only for existing apps
|
|
let appBackupsToRemove = [];
|
|
for (const appId of Object.keys(appBackupsById)) {
|
|
applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
|
|
appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
|
|
}
|
|
|
|
for (const appBackup of appBackupsToRemove) {
|
|
progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
|
|
removedAppBackupIds.push(appBackup.id);
|
|
await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
|
|
}
|
|
|
|
debug('cleanupAppBackups: done');
|
|
|
|
callback(null, removedAppBackupIds);
|
|
});
|
|
}
|
|
|
|
async function cleanupBoxBackups(backupConfig, progressCallback) {
|
|
assert.strictEqual(typeof backupConfig, 'object');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
let referencedAppBackupIds = [], removedBoxBackupIds = [];
|
|
|
|
const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
|
|
|
|
applyBackupRetentionPolicy(boxBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), [] /* references */);
|
|
|
|
for (const boxBackup of boxBackups) {
|
|
if (boxBackup.keepReason) {
|
|
referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
|
|
continue;
|
|
}
|
|
|
|
progressCallback({ message: `Removing box backup ${boxBackup.id}`});
|
|
|
|
removedBoxBackupIds.push(boxBackup.id);
|
|
await cleanupBackup(backupConfig, boxBackup, progressCallback);
|
|
}
|
|
|
|
debug('cleanupBoxBackups: done');
|
|
|
|
return { removedBoxBackupIds, referencedAppBackupIds };
|
|
}
|
|
|
|
async function cleanupMissingBackups(backupConfig, progressCallback) {
|
|
assert.strictEqual(typeof backupConfig, 'object');
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
|
|
const perPage = 1000;
|
|
let missingBackupIds = [];
|
|
const backupExists = util.promisify(storage.api(backupConfig.provider).exists);
|
|
|
|
if (constants.TEST) return missingBackupIds;
|
|
|
|
for (let page = 0, result = []; result.length < perPage; page++) {
|
|
result = await backups.list(page, perPage);
|
|
|
|
for (const backup of result) {
|
|
let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
|
|
if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
|
|
|
|
const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
|
|
if (existsError || exists) continue;
|
|
|
|
progressCallback({ message: `Removing missing backup ${backup.id}`});
|
|
|
|
const [delError] = await safe(backups.del(backup.id));
|
|
if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);
|
|
|
|
missingBackupIds.push(backup.id);
|
|
}
|
|
}
|
|
|
|
return missingBackupIds;
|
|
}
|
|
|
|
// removes the snapshots of apps that have been uninstalled
|
|
function cleanupSnapshots(backupConfig, callback) {
|
|
assert.strictEqual(typeof backupConfig, 'object');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
var contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
|
|
var info = safe.JSON.parse(contents);
|
|
if (!info) return callback();
|
|
|
|
delete info.box;
|
|
async.eachSeries(Object.keys(info), function (appId, iteratorDone) {
|
|
apps.get(appId, function (error /*, app */) {
|
|
if (!error || error.reason !== BoxError.NOT_FOUND) return iteratorDone();
|
|
|
|
function done(/* ignoredError */) {
|
|
safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
|
|
safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache.new`));
|
|
|
|
backups.setSnapshotInfo(appId, null, function (/* ignoredError */) {
|
|
debug('cleanupSnapshots: cleaned up snapshot of app id %s', appId);
|
|
|
|
iteratorDone();
|
|
});
|
|
}
|
|
|
|
if (info[appId].format ==='tgz') {
|
|
storage.api(backupConfig.provider).remove(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format), done);
|
|
} else {
|
|
var events = storage.api(backupConfig.provider).removeDir(backupConfig, storage.getBackupFilePath(backupConfig, `snapshot/app_${appId}`, info[appId].format));
|
|
events.on('progress', function (detail) { debug(`cleanupSnapshots: ${detail}`); });
|
|
events.on('done', done);
|
|
}
|
|
});
|
|
}, function () {
|
|
debug('cleanupSnapshots: done');
|
|
|
|
callback();
|
|
});
|
|
}
|
|
|
|
function run(progressCallback, callback) {
|
|
assert.strictEqual(typeof progressCallback, 'function');
|
|
assert.strictEqual(typeof callback, 'function');
|
|
|
|
settings.getBackupConfig(async function (error, backupConfig) {
|
|
if (error) return callback(error);
|
|
|
|
if (backupConfig.retentionPolicy.keepWithinSecs < 0) {
|
|
debug('cleanup: keeping all backups');
|
|
return callback(null, {});
|
|
}
|
|
|
|
progressCallback({ percent: 10, message: 'Cleaning box backups' });
|
|
|
|
const [cleanupBoxError, removedBoxBackups ] = await safe(cleanupBoxBackups(backupConfig, progressCallback));
|
|
if (cleanupBoxError) return callback(cleanupBoxError);
|
|
|
|
const { removedBoxBackupIds, referencedAppBackupIds } = removedBoxBackups;
|
|
|
|
progressCallback({ percent: 40, message: 'Cleaning app backups' });
|
|
|
|
cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback, async function (error, removedAppBackupIds) {
|
|
if (error) return callback(error);
|
|
|
|
progressCallback({ percent: 70, message: 'Cleaning missing backups' });
|
|
|
|
const [cleanupMissingBackupsError, missingBackupIds] = await cleanupMissingBackups(backupConfig, progressCallback);
|
|
if (cleanupMissingBackupsError) return callback(cleanupMissingBackupsError);
|
|
|
|
progressCallback({ percent: 90, message: 'Cleaning snapshots' });
|
|
|
|
cleanupSnapshots(backupConfig, function (error) {
|
|
if (error) return callback(error);
|
|
|
|
callback(null, { removedBoxBackupIds, removedAppBackupIds, missingBackupIds });
|
|
});
|
|
});
|
|
});
|
|
}
|