jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: add backup multiple targets

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-07-24 19:02:02 +02:00
parent 100bea981d
commit 3aafbd2ccb
25 changed files with 744 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backupcleaner.js
+41 -43
View File
@@ -80,20 +80,20 @@ function applyBackupRetention(allBackups, retention, referencedBackupIds) {
}
}
async function removeBackup(backupConfig, backup, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function removeBackup(target, backup, progressCallback) {
assert.strictEqual(typeof target, 'object');
assert.strictEqual(typeof backup, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const backupFilePath = backupFormat.api(backupConfig.format).getBackupFilePath(backupConfig, backup.remotePath);
const backupFilePath = backupFormat.api(target.format).getBackupFilePath(target, backup.remotePath);
let removeError;
if (backupConfig.format ==='tgz') {
if (target.format ==='tgz') {
progressCallback({ message: `${backup.remotePath}: Removing ${backupFilePath}`});
[removeError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFilePath));
[removeError] = await safe(storage.api(target.provider).remove(target.config, backupFilePath));
} else {
progressCallback({ message: `${backup.remotePath}: Removing directory ${backupFilePath}`});
[removeError] = await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback));
[removeError] = await safe(storage.api(target.provider).removeDir(target.config, backupFilePath, progressCallback));
}
if (removeError) {
@@ -102,7 +102,7 @@ async function removeBackup(backupConfig, backup, progressCallback) {
}
// prune empty directory if possible
const [pruneError] = await safe(storage.api(backupConfig.provider).remove(backupConfig, path.dirname(backupFilePath)));
const [pruneError] = await safe(storage.api(target.provider).remove(target.config, path.dirname(backupFilePath)));
if (pruneError) debug(`removeBackup: unable to prune backup directory ${path.dirname(backupFilePath)}: ${pruneError.message}`);
const [delError] = await safe(backupListing.del(backup.id));
@@ -110,9 +110,8 @@ async function removeBackup(backupConfig, backup, progressCallback) {
else debug(`removeBackup: removed ${backup.remotePath}`);
}
async function cleanupAppBackups(backupConfig, retention, referencedBackupIds, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof retention, 'object');
async function cleanupAppBackups(target, referencedBackupIds, progressCallback) {
assert.strictEqual(typeof target, 'object');
assert(Array.isArray(referencedBackupIds));
assert.strictEqual(typeof progressCallback, 'function');
@@ -134,7 +133,7 @@ async function cleanupAppBackups(backupConfig, retention, referencedBackupIds, p
// apply backup policy per app. keep latest backup only for existing apps
let appBackupsToRemove = [];
for (const appId of Object.keys(appBackupsById)) {
const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, retention);
const appRetention = Object.assign({ keepLatest: allAppIds.includes(appId) }, target.retention);
debug(`cleanupAppBackups: applying retention for appId ${appId} retention: ${JSON.stringify(appRetention)}`);
applyBackupRetention(appBackupsById[appId], appRetention, referencedBackupIds);
appBackupsToRemove = appBackupsToRemove.concat(appBackupsById[appId].filter(b => !b.keepReason));
@@ -143,7 +142,7 @@ async function cleanupAppBackups(backupConfig, retention, referencedBackupIds, p
for (const appBackup of appBackupsToRemove) {
await progressCallback({ message: `Removing app backup (${appBackup.identifier}): ${appBackup.id}`});
removedAppBackupPaths.push(appBackup.remotePath);
await removeBackup(backupConfig, appBackup, progressCallback); // never errors
await removeBackup(target, appBackup, progressCallback); // never errors
}
debug('cleanupAppBackups: done');
@@ -151,9 +150,8 @@ async function cleanupAppBackups(backupConfig, retention, referencedBackupIds, p
return removedAppBackupPaths;
}
async function cleanupMailBackups(backupConfig, retention, referencedBackupIds, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof retention, 'object');
async function cleanupMailBackups(target, referencedBackupIds, progressCallback) {
assert.strictEqual(typeof target, 'object');
assert(Array.isArray(referencedBackupIds));
assert.strictEqual(typeof progressCallback, 'function');
@@ -161,13 +159,13 @@ async function cleanupMailBackups(backupConfig, retention, referencedBackupIds,
const mailBackups = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_MAIL, 1, 100000);
applyBackupRetention(mailBackups, Object.assign({ keepLatest: true }, retention), referencedBackupIds);
applyBackupRetention(mailBackups, Object.assign({ keepLatest: true }, target.retention), referencedBackupIds);
for (const mailBackup of mailBackups) {
if (mailBackup.keepReason) continue;
await progressCallback({ message: `Removing mail backup ${mailBackup.remotePath}`});
removedMailBackupPaths.push(mailBackup.remotePath);
await removeBackup(backupConfig, mailBackup, progressCallback); // never errors
await removeBackup(target, mailBackup, progressCallback); // never errors
}
debug('cleanupMailBackups: done');
@@ -175,9 +173,7 @@ async function cleanupMailBackups(backupConfig, retention, referencedBackupIds,
return removedMailBackupPaths;
}
async function cleanupBoxBackups(backupConfig, retention, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof retention, 'object');
async function cleanupBoxBackups(target, progressCallback) {
assert.strictEqual(typeof progressCallback, 'function');
let referencedBackupIds = [];
@@ -188,7 +184,7 @@ async function cleanupBoxBackups(backupConfig, retention, progressCallback) {
// 100000 here should be seen as infinity
const boxBackups = await backupListing.getByTypePaged(backupListing.BACKUP_TYPE_BOX, 1, 100000);
applyBackupRetention(boxBackups, Object.assign({ keepLatest: true }, retention), [] /* references */);
applyBackupRetention(boxBackups, Object.assign({ keepLatest: true }, target.retention), [] /* references */);
for (const boxBackup of boxBackups) {
if (boxBackup.keepReason) {
@@ -199,7 +195,7 @@ async function cleanupBoxBackups(backupConfig, retention, progressCallback) {
await progressCallback({ message: `Removing box backup ${boxBackup.remotePath}`});
removedBoxBackupPaths.push(boxBackup.remotePath);
await removeBackup(backupConfig, boxBackup, progressCallback);
await removeBackup(target, boxBackup, progressCallback);
}
debug('cleanupBoxBackups: done');
@@ -208,8 +204,8 @@ async function cleanupBoxBackups(backupConfig, retention, progressCallback) {
}
// cleans up the database by checking if backup exists in the remote. this can happen if user had set some bucket policy
async function cleanupMissingBackups(backupConfig, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function cleanupMissingBackups(target, progressCallback) {
assert.strictEqual(typeof target, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const perPage = 1000;
@@ -224,10 +220,10 @@ async function cleanupMissingBackups(backupConfig, progressCallback) {
for (const backup of result) {
if (backup.state !== backupListing.BACKUP_STATE_NORMAL) continue; // note: errored and incomplete backups are cleaned up by the backup retention logic
let backupFilePath = backupFormat.api(backupConfig.format).getBackupFilePath(backupConfig, backup.remotePath);
if (backupConfig.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
let backupFilePath = backupFormat.api(target.format).getBackupFilePath(target, backup.remotePath);
if (target.format === 'rsync') backupFilePath = backupFilePath + '/'; // add trailing slash to indicate directory
const [existsError, exists] = await safe(storage.api(backupConfig.provider).exists(backupConfig, backupFilePath));
const [existsError, exists] = await safe(storage.api(target.provider).exists(target.config, backupFilePath));
if (existsError || exists) continue;
await progressCallback({ message: `Removing missing backup ${backup.remotePath}`});
@@ -247,8 +243,8 @@ async function cleanupMissingBackups(backupConfig, progressCallback) {
}
// removes the snapshots of apps that have been uninstalled
async function cleanupSnapshots(backupConfig) {
assert.strictEqual(typeof backupConfig, 'object');
async function cleanupSnapshots(backupTarget) {
assert.strictEqual(typeof backupTarget, 'object');
const contents = safe.fs.readFileSync(paths.SNAPSHOT_INFO_FILE, 'utf8');
const info = safe.JSON.parse(contents);
@@ -263,9 +259,9 @@ async function cleanupSnapshots(backupConfig) {
if (app) continue; // app is still installed
if (info[appId].format ==='tgz') {
await safe(storage.api(backupConfig.provider).remove(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`)), { debug });
await safe(storage.api(backupTarget.provider).remove(backupTarget.config, backupFormat.api(info[appId].format).getBackupFilePath(backupTarget, `snapshot/app_${appId}`)), { debug });
} else {
await safe(storage.api(backupConfig.provider).removeDir(backupConfig, backupFormat.api(info[appId].format).getBackupFilePath(backupConfig, `snapshot/app_${appId}`), progressCallback), { debug });
await safe(storage.api(backupTarget.provider).removeDir(backupTarget.config, backupFormat.api(info[appId].format).getBackupFilePath(backupTarget, `snapshot/app_${appId}`), progressCallback), { debug });
}
safe.fs.unlinkSync(path.join(paths.BACKUP_INFO_DIR, `${appId}.sync.cache`));
@@ -278,40 +274,42 @@ async function cleanupSnapshots(backupConfig) {
debug('cleanupSnapshots: done');
}
async function run(progressCallback) {
async function run(targetId, progressCallback) {
assert.strictEqual(typeof targetId, 'string');
assert.strictEqual(typeof progressCallback, 'function');
const backupConfig = await backupTargets.getConfig();
const { retention } = await backupTargets.getPolicy();
debug(`run: retention is ${JSON.stringify(retention)}`);
const target = await backupTargets.get(targetId);
if (!target) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Target not found');
const status = await backupTargets.ensureMounted();
debug(`run: retention is ${JSON.stringify(target.retention)}`);
const status = await backupTargets.ensureMounted(target);
debug(`run: mount point status is ${JSON.stringify(status)}`);
if (status.state !== 'active') throw new BoxError(BoxError.MOUNT_ERROR, `Backup endpoint is not mounted: ${status.message}`);
if (retention.keepWithinSecs < 0) {
if (target.retention.keepWithinSecs < 0) {
debug('run: keeping all backups');
return {};
}
await progressCallback({ percent: 10, message: 'Cleaning box backups' });
const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(backupConfig, retention, progressCallback); // references is app or mail backup ids
const { removedBoxBackupPaths, referencedBackupIds } = await cleanupBoxBackups(target, progressCallback); // references is app or mail backup ids
await progressCallback({ percent: 20, message: 'Cleaning mail backups' });
const removedMailBackupPaths = await cleanupMailBackups(backupConfig, retention, referencedBackupIds, progressCallback);
const removedMailBackupPaths = await cleanupMailBackups(target, referencedBackupIds, progressCallback);
await progressCallback({ percent: 40, message: 'Cleaning app backups' });
const archivedBackupIds = await archives.listBackupIds();
const removedAppBackupPaths = await cleanupAppBackups(backupConfig, retention, referencedBackupIds.concat(archivedBackupIds), progressCallback);
const removedAppBackupPaths = await cleanupAppBackups(target, referencedBackupIds.concat(archivedBackupIds), progressCallback);
await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' });
const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback);
const missingBackupPaths = await cleanupMissingBackups(target, progressCallback);
await progressCallback({ percent: 80, message: 'Cleaning snapshots' });
await cleanupSnapshots(backupConfig);
await cleanupSnapshots(target);
await progressCallback({ percent: 80, message: 'Cleaning storage artifacts' });
await storage.api(backupConfig.provider).cleanup(backupConfig, progressCallback);
await storage.api(target.provider).cleanup(target.config, progressCallback);
return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };
}