jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

storage: rework listDir api to be a generator (like) function

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-02-12 18:46:54 +01:00
parent da0dcf65b3
commit 9888aa8c08
9 changed files with 143 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/backupformat/rsync.js
View File

@@ -25,8 +25,7 @@ const assert = require('assert'),
shell = require('../shell.js')('backupformat/rsync'),
storage = require('../storage.js'),
stream = require('stream/promises'),
syncer = require('../syncer.js'),
util = require('util');
syncer = require('../syncer.js');
function getBackupFilePath(backupConfig, remotePath) {
assert.strictEqual(typeof backupConfig, 'object');
@@ -179,12 +178,11 @@ async function restoreFsMetadata(dataLayout, metadataFile) {
}
}
function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback, callback) {
async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backupFilePath, 'string');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}`);
@@ -236,12 +234,15 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
});
}
storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, function (entries, iteratorDone) {
// https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
const concurrency = backupConfig.limits?.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
async.eachLimit(entries, concurrency, downloadFile, iteratorDone);
}, callback);
// https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
const concurrency = backupConfig.limits?.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
let marker = null;
while (true) {
const batch = await storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, 1000, marker);
await async.eachLimit(batch.entries, concurrency, downloadFile);
if (!batch.marker) break;
marker = batch.marker;
}
}
async function download(backupConfig, remotePath, dataLayout, progressCallback) {
@@ -251,11 +252,10 @@ async function download(backupConfig, remotePath, dataLayout, progressCallback)
assert.strictEqual(typeof progressCallback, 'function');
const backupFilePath = getBackupFilePath(backupConfig, remotePath);
const downloadDirAsync = util.promisify(downloadDir);
debug(`download: Downloading ${backupFilePath} to ${dataLayout.toString()}`);
await downloadDirAsync(backupConfig, backupFilePath, dataLayout, progressCallback);
await downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback);
await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
}