syncer: clean up superfluous files

src/backupcleaner.js

@@ -6,6 +6,7 @@ import backups from './backups.js';
 import backupFormats from './backupformats.js';
 import backupSites from './backupsites.js';
 import constants from './constants.js';
+import consumers from 'node:stream/consumers';
 import debugModule from 'debug';
 import moment from 'moment';
 import path from 'node:path';
@@ -265,6 +266,50 @@ async function removeOldAppSnapshots(site) {
     debug('removeOldAppSnapshots: done');
 }
 
+// the rsync algo had a bug (2c12bee79) where it would miss on deleting files on the snapshot
+// which in turn makes the integrity checker complain
+async function cleanupSnapshotSuperfluous(site, progressCallback) {
+    assert.strictEqual(typeof site, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    if (site.format !== 'rsync') return;
+
+    const snapshotInfo = await backupSites.getSnapshotInfo(site);
+
+    for (const id of Object.keys(snapshotInfo)) {
+        const remotePath = (id === 'box' || id === 'mail') ? `snapshot/${id}` : `snapshot/app_${id}`;
+
+        const [downloadError, sourceStream] = await safe(backupSites.storageApi(site).download(site.config, `${remotePath}.backupinfo`));
+        if (downloadError) {
+            debug(`cleanupSnapshotSuperfluous: no backupinfo for ${remotePath}, skipping`);
+            continue;
+        }
+
+        const buffer = await consumers.buffer(sourceStream);
+        const backupInfo = JSON.parse(buffer.toString('utf8'));
+        const expectedFiles = new Set(Object.keys(backupInfo));
+
+        let marker = null, removed = 0;
+        while (true) {
+            const batch = await backupSites.storageApi(site).listDir(site.config, remotePath, 1000, marker);
+            for (const entry of batch.entries) {
+                const relativePath = path.relative(remotePath, entry.path);
+                if (!expectedFiles.has(relativePath)) {
+                    progressCallback({ message: `Removing superfluous file: ${entry.path}` });
+                    await backupSites.storageApi(site).remove(site.config, entry.path);
+                    ++removed;
+                }
+            }
+            if (!batch.marker) break;
+            marker = batch.marker;
+        }
+
+        if (removed) debug(`cleanupSnapshotSuperfluous: removed ${removed} superfluous files from ${remotePath}`);
+    }
+
+    debug('cleanupSnapshotSuperfluous: done');
+}
+
 async function run(siteId, progressCallback) {
     assert.strictEqual(typeof siteId, 'string');
     assert.strictEqual(typeof progressCallback, 'function');
@@ -299,7 +344,10 @@ async function run(siteId, progressCallback) {
     await progressCallback({ percent: 80, message: 'Removing snapshots of uninstalled apps' });
     await removeOldAppSnapshots(site);
 
-    await progressCallback({ percent: 80, message: 'Cleaning storage artifacts' });
+    await progressCallback({ percent: 85, message: 'Cleaning superfluous files from snapshots' });
+    await cleanupSnapshotSuperfluous(site, progressCallback);
+
+    await progressCallback({ percent: 90, message: 'Cleaning storage artifacts' });
     await backupSites.storageApi(site).cleanup(site.config, progressCallback);
 
     return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths };