jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add aggregated size to system backups

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-10-10 12:55:03 +02:00
parent d603ea50e2
commit 961959d361
2 changed files with 45 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backuptask.js
+27 -19
View File
@@ -33,7 +33,8 @@ const apps = require('./apps.js'),
safe = require('safetydance'),
services = require('./services.js'),
shell = require('./shell.js')('backuptask'),
stream = require('stream/promises');
stream = require('stream/promises'),
util = require('util');
const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js');
@@ -269,9 +270,9 @@ async function copy(backupSite, srcRemotePath, destRemotePath, progressCallback)
debug(`copy: copied backupinfo successfully to ${destRemotePath}.backupinfo`);
}
async function backupBox(backupSite, dependsOn, tag, options, progressCallback) {
async function backupBox(backupSite, appBackupsMap, tag, options, progressCallback) {
assert.strictEqual(typeof backupSite, 'object');
assert(Array.isArray(dependsOn));
assert(util.types.isMap(appBackupsMap), 'integrityMap should be a Map'); // id -> stats { fileCount, size, startTime, totalMsecs, transferred }
assert.strictEqual(typeof tag, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
@@ -282,6 +283,14 @@ async function backupBox(backupSite, dependsOn, tag, options, progressCallback)
debug(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}`);
stats.aggregated = Array.from(appBackupsMap.values()).reduce((acc, s) => ({
fileCount: acc.fileCount + s.fileCount,
size: acc.size + s.size,
startTime: Math.min(acc.startTime, s.startTime),
totalMsecs: acc.totalMsecs + s.totalMsecs,
transferred: acc.transferred + s.transferred,
}), stats);
const data = {
remotePath,
encryptionVersion: backupSite.encryption ? 2 : null,
@@ -289,7 +298,7 @@ async function backupBox(backupSite, dependsOn, tag, options, progressCallback)
type: backups.BACKUP_TYPE_BOX,
state: backups.BACKUP_STATE_CREATING,
identifier: backups.BACKUP_IDENTIFIER_BOX,
dependsOn,
dependsOn: [...appBackupsMap.keys()],
manifest: null,
preserveSecs: options.preserveSecs || 0,
appConfig: null,
@@ -397,7 +406,7 @@ async function backupAppWithTag(app, backupSite, tag, options, progressCallback)
await backups.setState(id, state);
if (error) throw error;
return id;
return { id, stats: data.stats };
}
async function backupApp(app, backupSite, options, progressCallback) {
@@ -406,17 +415,17 @@ async function backupApp(app, backupSite, options, progressCallback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
let backupId = null;
let backup = null;
await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
if (options.snapshotOnly) {
await snapshotApp(app, progressCallback);
} else {
const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
backupId = await backupAppWithTag(app, backupSite, tag, options, progressCallback);
backup = await backupAppWithTag(app, backupSite, tag, options, progressCallback);
}
await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
return backupId;
return backup;
}
async function uploadMailSnapshot(backupSite, progressCallback) {
@@ -485,7 +494,7 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
await backups.setState(id, state);
if (error) throw error;
return id;
return { id, stats: data.stats };
}
async function downloadMail(backupSite, remotePath, progressCallback) {
@@ -519,7 +528,7 @@ async function fullBackup(backupSiteId, options, progressCallback) {
let percent = 1;
const step = 100/(allApps.length+3);
const appBackupIds = [];
const appBackupsMap = new Map(); // id -> stats
for (let i = 0; i < allApps.length; i++) {
const app = allApps[i];
percent += step;
@@ -536,24 +545,24 @@ async function fullBackup(backupSiteId, options, progressCallback) {
progressCallback({ percent, message: `Backing up ${app.fqdn} (${i+1}/${allApps.length}). Waiting for lock` });
await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
const startTime = new Date();
const [appBackupError, appBackupId] = await safe(backupAppWithTag(app, backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message })));
const [appBackupError, appBackup] = await safe(backupAppWithTag(app, backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message })));
debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`);
if (appBackupError) throw appBackupError;
if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
if (appBackup) appBackupsMap.set(appBackup.id, appBackup.stats); // backupId can be null if in BAD_STATE and never backed up
}
if (!backupSites.hasContent(backupSite, 'box')) return appBackupIds;
if (!backupSites.hasContent(backupSite, 'box')) return [...appBackupsMap.keys()];
progressCallback({ percent, message: 'Backing up mail' });
percent += step;
const mailBackupId = await backupMailWithTag(backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
const mailBackup = await backupMailWithTag(backupSite, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
appBackupsMap.set(mailBackup.id, mailBackup.stats);
progressCallback({ percent, message: 'Backing up system data' });
percent += step;
const dependsOn = appBackupIds.concat(mailBackupId);
const backupId = await backupBox(backupSite, dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
const backupId = await backupBox(backupSite, appBackupsMap, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
return backupId;
}
@@ -572,8 +581,7 @@ async function appBackup(appId, backupSiteId, options, progressCallback) {
await progressCallback({ percent: 1, message: `Backing up ${app.fqdn}. Waiting for lock` });
const startTime = new Date();
const backupId = await backupApp(app, backupSite, options, progressCallback);
const backup = await backupApp(app, backupSite, options, progressCallback);
await progressCallback({ percent: 100, message: `app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds` });
return backupId;
return backup.id;
}