diff --git a/src/backupcleaner.js b/src/backupcleaner.js index c3fa40ebf..6d7b91a1f 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -6,6 +6,7 @@ import backups from './backups.js'; import backupFormats from './backupformats.js'; import backupSites from './backupsites.js'; import constants from './constants.js'; +import consumers from 'node:stream/consumers'; import debugModule from 'debug'; import moment from 'moment'; import path from 'node:path'; @@ -265,6 +266,50 @@ async function removeOldAppSnapshots(site) { debug('removeOldAppSnapshots: done'); } +// the rsync algo had a bug (2c12bee79) where it would miss on deleting files on the snapshot +// which in turn makes the integrity checker complain +async function cleanupSnapshotSuperfluous(site, progressCallback) { + assert.strictEqual(typeof site, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + if (site.format !== 'rsync') return; + + const snapshotInfo = await backupSites.getSnapshotInfo(site); + + for (const id of Object.keys(snapshotInfo)) { + const remotePath = (id === 'box' || id === 'mail') ? `snapshot/${id}` : `snapshot/app_${id}`; + + const [downloadError, sourceStream] = await safe(backupSites.storageApi(site).download(site.config, `${remotePath}.backupinfo`)); + if (downloadError) { + debug(`cleanupSnapshotSuperfluous: no backupinfo for ${remotePath}, skipping`); + continue; + } + + const buffer = await consumers.buffer(sourceStream); + const backupInfo = JSON.parse(buffer.toString('utf8')); + const expectedFiles = new Set(Object.keys(backupInfo)); + + let marker = null, removed = 0; + while (true) { + const batch = await backupSites.storageApi(site).listDir(site.config, remotePath, 1000, marker); + for (const entry of batch.entries) { + const relativePath = path.relative(remotePath, entry.path); + if (!expectedFiles.has(relativePath)) { + progressCallback({ message: `Removing superfluous file: ${entry.path}` }); + await backupSites.storageApi(site).remove(site.config, entry.path); + ++removed; + } + } + if (!batch.marker) break; + marker = batch.marker; + } + + if (removed) debug(`cleanupSnapshotSuperfluous: removed ${removed} superfluous files from ${remotePath}`); + } + + debug('cleanupSnapshotSuperfluous: done'); +} + async function run(siteId, progressCallback) { assert.strictEqual(typeof siteId, 'string'); assert.strictEqual(typeof progressCallback, 'function'); @@ -299,7 +344,10 @@ async function run(siteId, progressCallback) { await progressCallback({ percent: 80, message: 'Removing snapshots of uninstalled apps' }); await removeOldAppSnapshots(site); - await progressCallback({ percent: 80, message: 'Cleaning storage artifacts' }); + await progressCallback({ percent: 85, message: 'Cleaning superfluous files from snapshots' }); + await cleanupSnapshotSuperfluous(site, progressCallback); + + await progressCallback({ percent: 90, message: 'Cleaning storage artifacts' }); await backupSites.storageApi(site).cleanup(site.config, progressCallback); return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };