storage: make copy async

This commit is contained in:
Girish Ramakrishnan
2022-04-30 16:01:42 -07:00
parent 8ceb80dc44
commit ea01586b52
7 changed files with 54 additions and 95 deletions

View File

@@ -31,7 +31,6 @@ const assert = require('assert'),
constants = require('../constants.js'),
DataLayout = require('../datalayout.js'),
debug = require('debug')('box:storage/gcs'),
EventEmitter = require('events'),
PassThrough = require('stream').PassThrough,
path = require('path'),
safe = require('safetydance'),
@@ -184,12 +183,11 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
}, callback);
}
function copy(apiConfig, oldFilePath, newFilePath) {
async function copy(apiConfig, oldFilePath, newFilePath, progressCallback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof oldFilePath, 'string');
assert.strictEqual(typeof newFilePath, 'string');
var events = new EventEmitter();
assert.strictEqual(typeof progressCallback, 'function');
function copyFile(entry, iteratorCallback) {
var relativePath = path.relative(oldFilePath, entry.fullPath);
@@ -206,20 +204,19 @@ function copy(apiConfig, oldFilePath, newFilePath) {
const batchSize = 1000;
const concurrency = apiConfig.copyConcurrency || 10;
var total = 0;
let total = 0;
listDir(apiConfig, oldFilePath, batchSize, function (entries, done) {
const listDirAsync = util.promisify(listDir);
const [copyError] = await safe(listDirAsync(apiConfig, oldFilePath, batchSize, function (entries, done) {
total += entries.length;
events.emit('progress', `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`);
progressCallback({ message: `Copying ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });
async.eachLimit(entries, concurrency, copyFile, done);
}, function (error) {
events.emit('progress', `Copied ${total} files`);
process.nextTick(() => events.emit('done', error));
});
}));
return events;
progressCallback({ message: `Copied ${total} files with error: ${copyError}` });
}
async function remove(apiConfig, filename) {