diff --git a/src/backups.js b/src/backups.js index fb8750d43..92977baa7 100644 --- a/src/backups.js +++ b/src/backups.js @@ -309,6 +309,8 @@ function sync(backupConfig, backupId, dataDir, progressCallback, callback) { assert.strictEqual(typeof progressCallback, 'function'); assert.strictEqual(typeof callback, 'function'); + const concurrency = backupConfig.syncConcurrency || (backupConfig.provider === 's3' ? 200 : 10); + syncer.sync(dataDir, function processTask(task, iteratorCallback) { debug('sync: processing task: %j', task); // the empty task.path is special to signify the directory @@ -349,7 +351,7 @@ function sync(backupConfig, backupId, dataDir, progressCallback, callback) { }); } }, iteratorCallback); - }, backupConfig.syncConcurrency || 10 /* concurrency */, function (error) { + }, concurrency, function (error) { if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message)); callback(); @@ -531,8 +533,9 @@ function downloadDir(backupConfig, backupFilePath, destDir, progressCallback, ca api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, done) { // https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441 - const limit = backupConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; - async.eachLimit(entries, limit, downloadFile, done); + const concurrency = backupConfig.downloadConcurrency || (backupConfig.provider === 's3' ? 1000 : 100); + + async.eachLimit(entries, concurrency, downloadFile, done); }, callback); } diff --git a/src/storage/s3.js b/src/storage/s3.js index fd85cb93c..82b4db080 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -353,12 +353,12 @@ function copy(apiConfig, oldFilePath, newFilePath) { }); } - var total = 0, concurrency = 4; + var total = 0; + const concurrency = apiConfig.copyConcurrency || (apiConfig.provider === 's3' ? 500 : 10); listDir(apiConfig, oldFilePath, 1000, function listDirIterator(entries, done) { total += entries.length; - if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5); events.emit('progress', `Copying ${total-entries.length}-${total}. ${retryCount} errors so far. concurrency set to ${concurrency}`); retryCount = 0;