jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backup logs: make them much terse and concise

Browse Source 
these are making the rsync logs massive. instead resort to reporting
progress based on file count. there is also a heartbeat timer for
"stuck" or "long downloading" files, every minute.
This commit is contained in:
Girish Ramakrishnan
2026-03-12 16:52:32 +05:30
parent b16b57f38b
commit d57554a48c
4 changed files with 56 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backupformat/tgz.js
+10 -4
View File
@@ -66,7 +66,6 @@ function addEntryToPack(pack, header, options) {
debug(`addToPack: error adding ${header.name} ${header.type} ${error.message}`);
reject(new BoxError(BoxError.FS_ERROR, error.message));
} else {
debug(`addToPack: added ${header.name} ${header.type}`);
resolve();
}
}));
@@ -91,7 +90,6 @@ async function addPathToPack(pack, localPath, dataLayout) {
while (queue.length) {
// if (pack.destroyed || outStream.destroyed) break;
const dir = queue.shift();
debug(`tarPack: processing ${dir}`);
const [readdirError, entries] = await safe(fs.promises.readdir(dir, { withFileTypes: true }));
if (!entries) {
debug(`tarPack: skipping directory ${dir}: ${readdirError.message}`);
@@ -146,6 +144,9 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
});
ps.on('heartbeat', function ({ elapsed, transferred }) {
progressCallback({ message: `Still uploading backup (${elapsed}s, ${Math.round(transferred/1024/1024)}M)` });
});
// careful not to have async code between here and pipeline() for 'error' handling
const pack = tar.pack();
@@ -166,6 +167,7 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
if (error) break; // the pipeline will error and we will retry the whole packing all over
fileCount += stats.fileCount;
}
debug(`tarPack: packed ${fileCount} files`);
pack.finalize(); // harmless to call if already in error state
@@ -190,13 +192,14 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
const extract = tar.extract();
const now = new Date();
let entryCount = 0;
extract.on('entry', async function (header, entryStream, next) {
if (path.isAbsolute(header.name)) {
debug(`tarExtract: ignoring absolute path ${header.name}`);
return next();
}
++entryCount;
const abspath = dataLayout.toLocalPath(header.name);
debug(`tarExtract: ${header.name} ${header.size} ${header.type} to ${abspath}`);
let error = null;
if (header.type === 'directory') {
[error] = await safe(fs.promises.mkdir(abspath, { recursive: true, mode: 0o755 }));
@@ -217,7 +220,7 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
[error] = await safe(fs.promises.lutimes(abspath, now /* atime */, header.mtime)); // for dirs, mtime will get overwritten
next(error);
});
extract.on('finish', () => debug('tarExtract: extract finished'));
extract.on('finish', () => debug(`tarExtract: extracted ${entryCount} entries`));
const gunzip = zlib.createGunzip({});
const ps = new ProgressStream({ interval: 10000 });
@@ -226,6 +229,9 @@ async function tarExtract(inStream, dataLayout, encryption, progressCallback) {
if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
});
ps.on('heartbeat', function ({ elapsed, transferred }) {
progressCallback({ message: `Still downloading backup (${elapsed}s, ${Math.round(transferred/1024/1024)}M)` });
});
if (encryption) {
const decrypt = new DecryptStream(encryption);