jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

gcs: remove concurrency logic

Browse Source 
this is more complicated than necessary
This commit is contained in:
Girish Ramakrishnan
2019-10-18 18:38:37 -07:00
parent f6377fd1c6
commit 29ab3e91b3
1 changed files with 8 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/storage/gcs.js
View File

@@ -143,7 +143,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
var events = new EventEmitter(), retryCount = 0;
var events = new EventEmitter();
function copyFile(entry, iteratorCallback) {
var relativePath = path.relative(oldFilePath, entry.fullPath);
@@ -156,19 +156,15 @@ function copy(apiConfig, oldFilePath, newFilePath) {
iteratorCallback(null);
});
events.emit('progress', `Copying ${relativePath}...`);
}
const batchSize = -1;
var total = 0, concurrency = 4;
const batchSize = 1000, concurrency = 10;
var total = 0;
listDir(apiConfig, oldFilePath, batchSize, function (entries, done) {
total += entries.length;
if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`);
retryCount = 0;
events.emit('progress', `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`);
async.eachLimit(entries, concurrency, copyFile, done);
}, function (error) {
@@ -197,17 +193,15 @@ function removeDir(apiConfig, pathPrefix) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
var events = new EventEmitter(), retryCount = 0;
var events = new EventEmitter();
const batchSize = 1;
var total = 0, concurrency = 4;
const batchSize = 1000, concurrency = 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
var total = 0;
listDir(apiConfig, pathPrefix, batchSize, function (entries, done) {
total += entries.length;
if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
events.emit('progress', `${retryCount} errors. concurrency set to ${concurrency}`);
retryCount = 0;
events.emit('progress', `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`);
async.eachLimit(entries, concurrency, function (entry, iteratorCallback) {
remove(apiConfig, entry.fullPath, iteratorCallback);