diff --git a/CHANGES b/CHANGES index 84f657195..5ec6b1903 100644 --- a/CHANGES +++ b/CHANGES @@ -2917,3 +2917,5 @@ * notifications: email notification when cloudron update failed * node: update to 22.13.1 * docker: update to 27.5.1 +* s3: automatically abort old multipart uploads + diff --git a/src/backupcleaner.js b/src/backupcleaner.js index 564485ef3..79dea3e70 100644 --- a/src/backupcleaner.js +++ b/src/backupcleaner.js @@ -302,8 +302,11 @@ async function run(progressCallback) { await progressCallback({ percent: 70, message: 'Checking storage backend and removing stale entries in database' }); const missingBackupPaths = await cleanupMissingBackups(backupConfig, progressCallback); - await progressCallback({ percent: 90, message: 'Cleaning snapshots' }); + await progressCallback({ percent: 80, message: 'Cleaning snapshots' }); await cleanupSnapshots(backupConfig); + await progressCallback({ percent: 80, message: 'Cleaning storage artifacts' }); + await storage.api(backupConfig.provider).cleanup(backupConfig, progressCallback); + return { removedBoxBackupPaths, removedMailBackupPaths, removedAppBackupPaths, missingBackupPaths }; } diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 68af540ea..e0163ccf3 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -14,6 +14,8 @@ exports = module.exports = { remove, removeDir, + cleanup, + testConfig, removePrivateFields, injectPrivateFields @@ -216,6 +218,11 @@ function validateBackupTarget(folder) { return null; } +async function cleanup(apiConfig, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); +} + async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/storage/gcs.js b/src/storage/gcs.js index fcaddfcf4..4e50579e4 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -13,6 +13,8 @@ exports = module.exports = { remove, removeDir, + cleanup, + testConfig, removePrivateFields, injectPrivateFields, @@ -181,6 +183,11 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { progressCallback({ progress: `Deleted ${total} files` }); } +async function cleanup(apiConfig, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); +} + async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/storage/interface.js b/src/storage/interface.js index 363e8d0f0..eb43694f0 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -109,6 +109,14 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { throw new BoxError(BoxError.NOT_IMPLEMENTED, 'removeDir is not implemented'); } +async function cleanup(apiConfig, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + // Result: none + throw new BoxError(BoxError.NOT_IMPLEMENTED, 'cleanup is not implemented'); +} + async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/storage/noop.js b/src/storage/noop.js index c970b5bcf..636f3fdbd 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -13,6 +13,8 @@ exports = module.exports = { remove, removeDir, + cleanup, + testConfig, removePrivateFields, injectPrivateFields @@ -94,6 +96,11 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { debug(`removeDir: ${pathPrefix}`); } +async function cleanup(apiConfig, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); +} + async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); } diff --git a/src/storage/s3.js b/src/storage/s3.js index c8de6a852..84b4acfaa 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -13,6 +13,8 @@ exports = module.exports = { remove, removeDir, + cleanup, + testConfig, removePrivateFields, injectPrivateFields, @@ -356,7 +358,7 @@ async function copyFile(apiConfig, oldFilePath, newFilePath, entry, progressCall uploadedParts[index] = { ETag: part.CopyPartResult.ETag, PartNumber: partCopyParams.PartNumber }; })); - if (copyError) { // we must still recommend the user to set a AbortIncompleteMultipartUpload lifecycle rule + if (copyError) { const abortParams = { Bucket: apiConfig.bucket, Key: path.join(newFilePath, relativePath), @@ -480,6 +482,24 @@ async function removeDir(apiConfig, pathPrefix, progressCallback) { progressCallback({ message: `Removed ${total} files` }); } +// often, the AbortIncompleteMultipartUpload lifecycle rule is not added to the bucket resulting in large bucket sizes over time +async function cleanup(apiConfig, progressCallback) { + assert.strictEqual(typeof apiConfig, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + const s3 = createS3Client(apiConfig, { retryStrategy: RETRY_STRATEGY }); + + const uploads = await s3.listMultipartUploads({ Bucket: apiConfig.bucket, Prefix: apiConfig.prefix }); + progressCallback({ message: `Cleaning up any aborted multi-part uploads. count:${uploads.Uploads?.length || 0} truncated:${uploads.IsTruncated}` }); + if (!uploads.Uploads) return; + + for (const upload of uploads.Uploads) { + if (Date.now() - new Date(upload.Initiated) < 3 * 24 * 60 * 60 * 1000) continue; // 3 days ago + progressCallback({ message: `Cleaning up multi-part upload uploadId:${upload.UploadId} key:${upload.Key}` }); + await safe(s3.abortMultipartUpload({ Bucket: apiConfig.bucket, Key: upload.Key, UploadId: upload.UploadId }), { debug }); // ignore error + } +} + async function testConfig(apiConfig) { assert.strictEqual(typeof apiConfig, 'object');