jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

rsync: throttle log messages during download

Browse Source
This commit is contained in:
Girish Ramakrishnan
2026-03-12 13:47:26 +05:30
parent 12177446a2
commit b16b57f38b
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backupformat/rsync.js
+13 -7
View File
@@ -175,7 +175,6 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
}
async function processSyncerChange(change) {
debug('sync: processing task: %j', change);
// the empty task.path is special to signify the directory
const destPath = change.path && backupSite.encryption?.encryptedFilenames ? hush.encryptFilePath(change.path, backupSite.encryption) : change.path;
const fullPath = path.join(remotePath, destPath);
@@ -226,6 +225,14 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
debug(`downloadDir: ${remotePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames}. encrypted files: ${!!backupSite.encryption}`);
let lastProgressTime = 0;
function throttledProgressCallback(progress) {
const now = Date.now();
if (now - lastProgressTime < 5000) return;
lastProgressTime = now;
progressCallback(progress);
}
async function downloadFile(entry) {
let relativePath = path.relative(remotePath, entry.path);
if (encryptedFilenames) {
@@ -239,19 +246,19 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, mkdirError.message);
await promiseRetry({ times: 3, interval: 20000 }, async function () {
progressCallback({ message: `Downloading ${entry.path} to ${destFilePath}` });
throttledProgressCallback({ message: `Downloading ${entry.path} to ${destFilePath}` });
const [downloadError, sourceStream] = await safe(backupSites.storageApi(backupSite).download(backupSite.config, entry.path));
if (downloadError) {
progressCallback({ message: `Download ${entry.path} to ${destFilePath} errored: ${downloadError.message}` });
debug(`downloadDir: download ${entry.path} to ${destFilePath} errored: ${downloadError.message}`);
throw downloadError;
}
const ps = new ProgressStream({ interval: 10000 }); // display a progress every 10 seconds
ps.on('progress', function (progress) {
const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.path}` }); // 0M@0MBps looks wrong
progressCallback({ message: `Downloading ${entry.path}: ${transferred}M@${speed}MBps` });
if (!transferred && !speed) return throttledProgressCallback({ message: `Downloading ${entry.path}` }); // 0M@0MBps looks wrong
throttledProgressCallback({ message: `Downloading ${entry.path}: ${transferred}M@${speed}MBps` });
});
const destStream = fs.createWriteStream(destFilePath);
@@ -267,10 +274,9 @@ async function downloadDir(backupSite, remotePath, dataLayout, progressCallback)
const [pipelineError] = await safe(pipeline(streams));
if (pipelineError) {
progressCallback({ message: `Download error ${entry.path} to ${destFilePath}: ${pipelineError.message}` });
debug(`downloadDir: download error ${entry.path} to ${destFilePath}: ${pipelineError.message}`);
throw pipelineError;
}
progressCallback({ message: `Download finished ${entry.path} to ${destFilePath}` });
});
}