jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: add remotePath

Browse Source 
the main motivation is that id can be used in REST API routes. previously,
the id was a path and this had a "/" in it. This made /api/v1/backups/:backupId
not work.
This commit is contained in:
Girish Ramakrishnan
2022-04-04 14:13:27 -07:00
parent 54934c41a7
commit 452a4d9a75
15 changed files with 234 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
src/backupcleaner.js
View File

@@ -81,27 +81,27 @@ function applyBackupRetentionPolicy(allBackups, policy, referencedBackupIds) {
}
}
async function cleanupBackup(backupConfig, backup, progressCallback) {
async function removeBackup(backupConfig, backup, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backup, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
const backupFilePath = storage.getBackupFilePath(backupConfig, backup.remotePath, backup.format);
return new Promise((resolve) => {
function done(error) {
if (error) {
debug('cleanupBackup: error removing backup %j : %s', backup, error.message);
debug('removeBackup: error removing backup %j : %s', backup, error.message);
return resolve();
}
// prune empty directory if possible
storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath), async function (error) {
if (error) debug('cleanupBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
if (error) debug('removeBackup: unable to prune backup directory %s : %s', path.dirname(backupFilePath), error.message);
const [delError] = await safe(backups.del(backup.id));
if (delError) debug('cleanupBackup: error removing from database', delError);
else debug('cleanupBackup: removed %s', backup.id);
if (delError) debug(`removeBackup: error removing ${backup.id} from database`, delError);
else debug(`removeBackup: removed ${backup.remotePath}`);
resolve();
});
@@ -112,18 +112,18 @@ async function cleanupBackup(backupConfig, backup, progressCallback) {
storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, done);
} else {
const events = storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath);
events.on('progress', (message) => progressCallback({ message: `${backup.id}: ${message}` }));
events.on('progress', (message) => progressCallback({ message: `${backup.remotePath}: ${message}` }));
events.on('done', done);
}
});
}
async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback) {
async function cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert(Array.isArray(referencedAppBackupIds));
assert(Array.isArray(referencedBackupIds));
assert.strictEqual(typeof progressCallback, 'function');
let removedAppBackupIds = [];
const removedAppBackupPaths = [];
const allApps = await apps.list();
const allAppIds = allApps.map(a => a.id);
@@ -140,49 +140,49 @@ async function cleanupAppBackups(backupConfig, referencedAppBackupIds, progressC
// apply backup policy per app. keep latest backup only for existing apps
let appBackupsToRemove = [];
for (const appId of Object.keys(appBackupsById)) {
applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedAppBackupIds);
applyBackupRetentionPolicy(appBackupsById[appId], _.extend({ keepLatest: allAppIds.includes(appId) }, backupConfig.retentionPolicy), referencedBackupIds);
appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
}
for (const appBackup of appBackupsToRemove) {
await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
removedAppBackupIds.push(appBackup.id);
await cleanupBackup(backupConfig, appBackup, progressCallback); // never errors
removedAppBackupPaths.push(appBackup.remotePath);
await removeBackup(backupConfig, appBackup, progressCallback); // never errors
}
debug('cleanupAppBackups: done');
return removedAppBackupIds;
return removedAppBackupPaths;
}
async function cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback) {
async function cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert(Array.isArray(referencedAppBackupIds));
assert(Array.isArray(referencedBackupIds));
assert.strictEqual(typeof progressCallback, 'function');
let removedMailBackupIds = [];
const removedMailBackupPaths = [];
const mailBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_MAIL, 1, 1000);
applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedAppBackupIds);
applyBackupRetentionPolicy(mailBackups, _.extend({ keepLatest: true }, backupConfig.retentionPolicy), referencedBackupIds);
for (const mailBackup of mailBackups) {
if (mailBackup.keepReason) continue;
await progressCallback({ message: `Removing mail backup ${mailBackup.id}`});
removedMailBackupIds.push(mailBackup.id);
await cleanupBackup(backupConfig, mailBackup, progressCallback); // never errors
await progressCallback({ message: `Removing mail backup ${mailBackup.remotePath}`});
removedMailBackupPaths.push(mailBackup.remotePath);
await removeBackup(backupConfig, mailBackup, progressCallback); // never errors
}
debug('cleanupMailBackups: done');
return removedMailBackupIds;
return removedMailBackupPaths;
}
async function cleanupBoxBackups(backupConfig, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof progressCallback, 'function');
let referencedAppBackupIds = [], removedBoxBackupIds = [];
let referencedBackupIds = [], removedBoxBackupPaths = [];
const boxBackups = await backups.getByTypePaged(backups.BACKUP_TYPE_BOX, 1, 1000);
@@ -190,48 +190,49 @@ async function cleanupBoxBackups(backupConfig, progressCallback) {
for (const boxBackup of boxBackups) {
if (boxBackup.keepReason) {
referencedAppBackupIds = referencedAppBackupIds.concat(boxBackup.dependsOn);
referencedBackupIds = referencedBackupIds.concat(boxBackup.dependsOn);
continue;
}
await progressCallback({ message: `Removing box backup ${boxBackup.id}`});
await progressCallback({ message: `Removing box backup ${boxBackup.remotePath}`});
removedBoxBackupIds.push(boxBackup.id);
await cleanupBackup(backupConfig, boxBackup, progressCallback);
removedBoxBackupPaths.push(boxBackup.remotePath);
await removeBackup(backupConfig, boxBackup, progressCallback);
}
debug('cleanupBoxBackups: done');
return { removedBoxBackupIds, referencedAppBackupIds };
return { removedBoxBackupPaths, referencedBackupIds };
}
// cleans up the database by checking if backup existsing in the remote
async function cleanupMissingBackups(backupConfig, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const perPage = 1000;
let missingBackupIds = [];
let missingBackupPaths = [];
const backupExists = util.promisify(storage.api(backupConfig.provider).exists);
if (constants.TEST) return missingBackupIds;
if (constants.TEST) return missingBackupPaths;
let page = 1, result = [];
do {
result = await backups.list(page, perPage);
for (const backup of result) {
let backupFilePath = storage.getBackupFilePath(backupConfig, backup.id, backup.format);
let backupFilePath = storage.getBackupFilePath(backupConfig, backup.remotePath, backup.format);
if (backup.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
const [existsError, exists] = await safe(backupExists(backupConfig, backupFilePath));
if (existsError || exists) continue;
await progressCallback({ message: `Removing missing backup ${backup.id}`});
await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
const [delError] = await safe(backups.del(backup.id));
if (delError) debug(`cleanupBackup: error removing ${backup.id} from database`, delError);
if (delError) debug(`cleanupMissingBackups: error removing ${backup.id} from database`, delError);
missingBackupIds.push(backup.id);
missingBackupPaths.push(backup.remotePath);
}
++ page;
@@ -239,7 +240,7 @@ async function cleanupMissingBackups(backupConfig, progressCallback) {
debug('cleanupMissingBackups: done');
return missingBackupIds;
return missingBackupPaths;
}
// removes the snapshots of apps that have been uninstalled
@@ -298,19 +299,19 @@ async function run(progressCallback) {
}
await progressCallback({ percent: 10, message: 'Cleaning box backups' });
const { removedBoxBackupIds, referencedAppBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback);
const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, progressCallback); // references is app or mail backup ids
await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
const removedMailBackupIds = await cleanupMailBackups(backupConfig, referencedAppBackupIds, progressCallback);
const removedMailBackupPaths = await cleanupMailBackups(backupConfig, referencedBackupIds, progressCallback);
await progressCallback({ percent: 40, message: 'Cleaning app backups' });
const removedAppBackupIds = await cleanupAppBackups(backupConfig, referencedAppBackupIds, progressCallback);
const removedAppBackupPaths = await cleanupAppBackups(backupConfig, referencedBackupIds, progressCallback);
await progressCallback({ percent: 70, message: 'Cleaning missing backups' });
const missingBackupIds = await cleanupMissingBackups(backupConfig, progressCallback);
const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);
await progressCallback({ percent: 90, message: 'Cleaning snapshots' });
await cleanupSnapshots(backupConfig);
return { removedBoxBackupIds, removedMailBackupIds, removedAppBackupIds, missingBackupIds };
return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };
}