diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js index 8147a5d01..33d45657b 100644 --- a/src/backupformat/rsync.js +++ b/src/backupformat/rsync.js @@ -27,14 +27,14 @@ const assert = require('assert'), stream = require('stream/promises'), syncer = require('../syncer.js'); -function getBackupFilePath(backupConfig, remotePath) { - assert.strictEqual(typeof backupConfig, 'object'); +function getBackupFilePath(backupTarget, remotePath) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); // we don't have a rootPath for noop - if (backupConfig.provider === 'noop') return remotePath; + if (backupTarget.provider === 'noop') return remotePath; - return path.join(backupConfig.rootPath, remotePath); + return path.join(backupTarget.config.rootPath, remotePath); } async function addFile(sourceFile, encryption, uploader, progressCallback) { @@ -77,46 +77,46 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) { await uploader.finish(); } -async function processSyncerChange(change, backupConfig, remotePath, dataLayout, progressCallback) { +async function processSyncerChange(change, backupTarget, remotePath, dataLayout, progressCallback) { debug('sync: processing task: %j', change); // the empty task.path is special to signify the directory - const destPath = change.path && backupConfig.encryptedFilenames ? hush.encryptFilePath(change.path, backupConfig.encryption) : change.path; - const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath); + const destPath = change.path && backupTarget.encryption?.encryptedFilenames ? hush.encryptFilePath(change.path, backupTarget.encryption) : change.path; + const backupFilePath = path.join(getBackupFilePath(backupTarget, remotePath), destPath); if (change.operation === 'removedir') { debug(`Removing directory ${backupFilePath}`); - await storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback); + await storage.api(backupTarget.provider).removeDir(backupTarget.config, backupFilePath, progressCallback); } else if (change.operation === 'remove') { debug(`Removing ${backupFilePath}`); - await storage.api(backupConfig.provider).remove(backupConfig, backupFilePath); + await storage.api(backupTarget.provider).remove(backupTarget.config, backupFilePath); } else if (change.operation === 'add') { await promiseRetry({ times: 5, interval: 20000, debug }, async (retryCount) => { progressCallback({ message: `Adding ${change.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '') }); debug(`Adding ${change.path} position ${change.position} try ${retryCount}`); - const uploader = await storage.api(backupConfig.provider).upload(backupConfig, backupFilePath); - await addFile(dataLayout.toLocalPath('./' + change.path), backupConfig.encryption, uploader, progressCallback); + const uploader = await storage.api(backupTarget.provider).upload(backupTarget.config, backupFilePath); + await addFile(dataLayout.toLocalPath('./' + change.path), backupTarget.encryption, uploader, progressCallback); }); } } -async function sync(backupConfig, remotePath, dataLayout, progressCallback) { - assert.strictEqual(typeof backupConfig, 'object'); +async function sync(backupTarget, remotePath, dataLayout, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof progressCallback, 'function'); // the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB - const concurrency = backupConfig.limits?.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10); + const concurrency = backupTarget.limits?.syncConcurrency || (backupTarget.provider === 's3' ? 20 : 10); const changes = await syncer.sync(dataLayout); debug(`sync: processing ${changes.delQueue.length} deletes and ${changes.addQueue.length} additions`); - const [delError] = await safe(async.eachLimit(changes.delQueue, concurrency, async (change) => await processSyncerChange(change, backupConfig, remotePath, dataLayout, progressCallback))); + const [delError] = await safe(async.eachLimit(changes.delQueue, concurrency, async (change) => await processSyncerChange(change, backupTarget, remotePath, dataLayout, progressCallback))); debug('sync: done processing deletes. error: %o', delError); if (delError) throw delError; - const [addError] = await safe(async.eachLimit(changes.addQueue, concurrency, async (change) => await processSyncerChange(change, backupConfig, remotePath, dataLayout, progressCallback))); + const [addError] = await safe(async.eachLimit(changes.addQueue, concurrency, async (change) => await processSyncerChange(change, backupTarget, remotePath, dataLayout, progressCallback))); debug('sync: done processing adds. error: %o', addError); if (addError) throw addError; @@ -191,18 +191,20 @@ async function restoreFsMetadata(dataLayout, metadataFile) { } } -async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback) { - assert.strictEqual(typeof backupConfig, 'object'); +async function downloadDir(backupTarget, backupFilePath, dataLayout, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof backupFilePath, 'string'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof progressCallback, 'function'); - debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}. encryption filenames: ${backupConfig.encryptedFilenames} content: ${!!backupConfig.encryption}`); + const encryptedFilenames = backupTarget.encryption?.encryptedFilenames || false; + + debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames} content: ${!!backupTarget.encryption}`); async function downloadFile(entry) { let relativePath = path.relative(backupFilePath, entry.fullPath); - if (backupConfig.encryptedFilenames) { - const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption); + if (encryptedFilenames) { + const { error, result } = hush.decryptFilePath(relativePath, backupTarget.encryption); if (error) throw new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file'); relativePath = result; } @@ -212,7 +214,7 @@ async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCal if (mkdirError) throw new BoxError(BoxError.FS_ERROR, mkdirError.message); await promiseRetry({ times: 3, interval: 20000 }, async function () { - const [downloadError, sourceStream] = await safe(storage.api(backupConfig.provider).download(backupConfig, entry.fullPath)); + const [downloadError, sourceStream] = await safe(storage.api(backupTarget.provider).download(backupTarget.config, entry.fullPath)); if (downloadError) { progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${downloadError.message}` }); throw downloadError; @@ -229,8 +231,8 @@ async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCal const streams = [ sourceStream, ps ]; - if (backupConfig.encryption) { - const decryptStream = new DecryptStream(backupConfig.encryption); + if (backupTarget.encryption) { + const decryptStream = new DecryptStream(backupTarget.encryption); streams.push(decryptStream); } @@ -248,36 +250,36 @@ async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCal } // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441 - const concurrency = backupConfig.limits?.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10); + const concurrency = backupTarget.limits?.downloadConcurrency || (backupTarget.provider === 's3' ? 30 : 10); let marker = null; while (true) { - const batch = await storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, marker === null ? 1 : 1000, marker); // try with one file first. if that works out, we continue faster + const batch = await storage.api(backupTarget.provider).listDir(backupTarget.config, backupFilePath, marker === null ? 1 : 1000, marker); // try with one file first. if that works out, we continue faster await async.eachLimit(batch.entries, concurrency, downloadFile); if (!batch.marker) break; marker = batch.marker; } } -async function download(backupConfig, remotePath, dataLayout, progressCallback) { - assert.strictEqual(typeof backupConfig, 'object'); +async function download(backupTarget, remotePath, dataLayout, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout'); assert.strictEqual(typeof progressCallback, 'function'); - const backupFilePath = getBackupFilePath(backupConfig, remotePath); + const backupFilePath = getBackupFilePath(backupTarget, remotePath); debug(`download: Downloading ${backupFilePath} to ${dataLayout.toString()}`); - await downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback); + await downloadDir(backupTarget, backupFilePath, dataLayout, progressCallback); await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); } -async function upload(backupConfig, remotePath, dataLayout, progressCallback) { - assert.strictEqual(typeof backupConfig, 'object'); +async function upload(backupTarget, remotePath, dataLayout, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof dataLayout, 'object'); assert.strictEqual(typeof progressCallback, 'function'); await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); - await sync(backupConfig, remotePath, dataLayout, progressCallback); + await sync(backupTarget, remotePath, dataLayout, progressCallback); }