storage: make remove and removeDir async

This commit is contained in:
Girish Ramakrishnan
2022-04-14 16:07:01 -05:00
parent 8d8cdd38a9
commit 685bda35b9
7 changed files with 88 additions and 139 deletions

View File

@@ -34,7 +34,8 @@ const assert = require('assert'),
EventEmitter = require('events'),
PassThrough = require('stream').PassThrough,
path = require('path'),
safe = require('safetydance');
safe = require('safetydance'),
util = require('util');
let GCS = require('@google-cloud/storage').Storage;
@@ -221,44 +222,32 @@ function copy(apiConfig, oldFilePath, newFilePath) {
return events;
}
function remove(apiConfig, filename, callback) {
async function remove(apiConfig, filename) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof filename, 'string');
assert.strictEqual(typeof callback, 'function');
getBucket(apiConfig)
.file(filename)
.delete(function (error) {
if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message);
callback(null);
});
const [error] = await safe(getBucket(apiConfig).file(filename).delete());
if (error) debug('removeBackups: Unable to remove %s (%s). Not fatal.', filename, error.message);
}
function removeDir(apiConfig, pathPrefix) {
async function removeDir(apiConfig, pathPrefix, progressCallback) {
assert.strictEqual(typeof apiConfig, 'object');
assert.strictEqual(typeof pathPrefix, 'string');
var events = new EventEmitter();
assert.strictEqual(typeof progressCallback, 'function');
const batchSize = 1000, concurrency = apiConfig.deleteConcurrency || 10; // https://googleapis.dev/nodejs/storage/latest/Bucket.html#deleteFiles
var total = 0;
let total = 0;
listDir(apiConfig, pathPrefix, batchSize, function (entries, done) {
const listDirAsync = util.promisify(listDir);
await listDirAsync(apiConfig, pathPrefix, batchSize, function (entries, done) {
total += entries.length;
events.emit('progress', `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}`);
progressCallback({ message: `Removing ${entries.length} files from ${entries[0].fullPath} to ${entries[entries.length-1].fullPath}. total: ${total}` });
async.eachLimit(entries, concurrency, function (entry, iteratorCallback) {
remove(apiConfig, entry.fullPath, iteratorCallback);
}, done);
}, function (error) {
events.emit('progress', `Deleted ${total} files`);
process.nextTick(() => events.emit('done', error));
async.eachLimit(entries, concurrency, async (entry) => await remove(apiConfig, entry.fullPath), done);
});
return events;
progressCallback({ progress: `Deleted ${total} files` });
}
async function remount(apiConfig) {