diff --git a/src/storage/gcs.js b/src/storage/gcs.js index 1ed72cf6f..8151de7f1 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -143,7 +143,7 @@ function copy(apiConfig, oldFilePath, newFilePath) { assert.strictEqual(typeof oldFilePath, 'string'); assert.strictEqual(typeof newFilePath, 'string'); - var events = new EventEmitter(), retryCount = 0; + var events = new EventEmitter(); function copyFile(entry, iteratorCallback) { var relativePath = path.relative(oldFilePath, entry.fullPath); @@ -156,19 +156,15 @@ function copy(apiConfig, oldFilePath, newFilePath) { iteratorCallback(null); }); - - events.emit('progress', `Copying ${relativePath}...`); } - const batchSize = -1; - var total = 0, concurrency = 4; + const batchSize = 1000, concurrency = 10; + var total = 0; listDir(apiConfig, oldFilePath, batchSize, function (entries, done) { total += entries.length; - if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5); - events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`); - retryCount = 0; + events.emit('progress', `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`); async.eachLimit(entries, concurrency, copyFile, done); }, function (error) { @@ -197,17 +193,15 @@ function removeDir(apiConfig, pathPrefix) { assert.strictEqual(typeof apiConfig, 'object'); assert.strictEqual(typeof pathPrefix, 'string'); - var events = new EventEmitter(), retryCount = 0; + var events = new EventEmitter(); - const batchSize = 1; - var total = 0, concurrency = 4; + const batchSize = 1000, concurrency = 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles + var total = 0; listDir(apiConfig, pathPrefix, batchSize, function (entries, done) { total += entries.length; - if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5); - events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`); - retryCount = 0; + events.emit('progress', `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`); async.eachLimit(entries, concurrency, function (entry, iteratorCallback) { remove(apiConfig, entry.fullPath, iteratorCallback);