diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js
index f9a28095a..edf188ec9 100644
--- a/src/backupformat/rsync.js
+++ b/src/backupformat/rsync.js
@@ -71,7 +71,7 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
await uploader.finish();
return {
- stats: ps.stats(), // { startTime, totalMsecs, transferred }
+ stats: { transferred: ps.stats().transferred },
integrity: { size: ps.stats().transferred, sha256: hash.digest('hex') }
};
}
@@ -91,8 +91,6 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
transferred: 0,
size: [...integrityMap.values()].reduce((sum, integrity) => sum + (integrity?.size || 0), 0), // integrity can be null if file had disappeared during upload
fileCount: addQueue.length + integrityMap.size, // final file count, not the transferred file count
- startTime: Date.now(),
- totalMsecs: 0
};
const destPathIntegrityMap = new Map(); // unlike integrityMap which contains local filenames, this contains destination filenames (maybe encrypted)
@@ -138,7 +136,7 @@ async function sync(backupSite, remotePath, dataLayout, progressCallback) {
await syncer.finalize(integrityMap, cacheFile);
return {
- stats: { ...aggregatedStats, totalMsecs: Date.now()-aggregatedStats.startTime },
+ stats: aggregatedStats,
integrityMap: destPathIntegrityMap
};
}
diff --git a/src/backupformat/tgz.js b/src/backupformat/tgz.js
index 2e6eff2a9..882e1aa0c 100644
--- a/src/backupformat/tgz.js
+++ b/src/backupformat/tgz.js
@@ -170,12 +170,12 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) {
const [error] = await pipeline; // already wrapped in safe()
if (error) throw new BoxError(BoxError.EXTERNAL_ERROR, `tarPack pipeline error: ${error.message}`);
- const stats = ps.stats();
+ const stats = ps.stats(); // { startTime, totalMsecs, transferred }
debug(`tarPack: pipeline finished: ${JSON.stringify(stats)}`);
await uploader.finish();
return {
- stats: { fileCount, size: stats.transferred, ...stats },
+ stats: { fileCount, size: stats.transferred, transferred: stats.transferred },
integrity: { size: stats.transferred, fileCount, sha256: hash.digest('hex') }
};
}
diff --git a/src/backups.js b/src/backups.js
index f2be478b4..a31542e3a 100644
--- a/src/backups.js
+++ b/src/backups.js
@@ -6,7 +6,6 @@ exports = module.exports = {
getLatestInTargetByIdentifier, // brutal function name
add,
update,
- setState,
list,
listBySiteId,
listByTypePaged,
@@ -147,9 +146,12 @@ async function update(backup, data) {
const fields = [], values = [];
for (const p in data) {
- if (p === 'label' || p === 'preserveSecs') {
+ if (p === 'label' || p === 'preserveSecs' || p === 'state') {
fields.push(p + ' = ?');
values.push(data[p]);
+ } else if (p === 'stats') {
+ fields.push(`${p}Json=?`);
+ values.push(JSON.stringify(data[p]));
}
}
values.push(backup.id);
@@ -165,14 +167,6 @@ async function update(backup, data) {
}
}
-async function setState(id, state) {
- assert.strictEqual(typeof id, 'string');
- assert.strictEqual(typeof state, 'string');
-
- const result = await database.query('UPDATE backups SET state = ? WHERE id = ?', [state, id]);
- if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found');
-}
-
async function list(page, perPage) {
assert(typeof page === 'number' && page > 0);
assert(typeof perPage === 'number' && perPage > 0);
diff --git a/src/backuptask.js b/src/backuptask.js
index e24a7fb5b..5b6df8183 100644
--- a/src/backuptask.js
+++ b/src/backuptask.js
@@ -106,7 +106,7 @@ async function upload(remotePath, siteId, dataLayoutString, progressCallback) {
// - tgz: only one entry named "." in the map. fileCount has the file count inside.
// - rsync: entry for each relative path.
// integrity - { signature } of the uploaded .backupinfo .
- // stats - { fileCount, size, startTime, totalMsecs, transferred }
+ // stats - { fileCount, size, transferred }
// - tgz: size (backup size) and transferred is the same
// - rsync: size (final backup size) will be different from what was transferred (only changed files)
// stats.fileCount and stats.size are stored in db and should match up what is written into .backupinfo
@@ -273,23 +273,25 @@ async function copy(backupSite, srcRemotePath, destRemotePath, progressCallback)
async function backupBox(backupSite, appBackupsMap, tag, options, progressCallback) {
assert.strictEqual(typeof backupSite, 'object');
- assert(util.types.isMap(appBackupsMap), 'integrityMap should be a Map'); // id -> stats { fileCount, size, startTime, totalMsecs, transferred }
+ assert(util.types.isMap(appBackupsMap), 'appBackupsMap should be a Map'); // id -> stats: { upload: { fileCount, size, startTime, duration, transferred } }
assert.strictEqual(typeof tag, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
- const { stats, integrity } = await uploadBoxSnapshot(backupSite, progressCallback);
+ const uploadStartTime = Date.now();
+ const uploadResult = await uploadBoxSnapshot(backupSite, progressCallback); // { stats, integrity }
+ const stats = { upload: { ...uploadResult.stats, startTime: uploadStartTime, duration: Date.now() - uploadStartTime } };
const remotePath = addFileExtension(backupSite, `${tag}/box_v${constants.VERSION}`);
// stats object might be null for stopped/errored apps from old versions
- stats.aggregated = Array.from(appBackupsMap.values()).filter(s => !!s).reduce((acc, s) => ({
- fileCount: acc.fileCount + s.fileCount,
- size: acc.size + s.size,
- startTime: Math.min(acc.startTime, s.startTime),
- totalMsecs: acc.totalMsecs + s.totalMsecs,
- transferred: acc.transferred + s.transferred,
- }), stats);
+ stats.aggregatedUpload = Array.from(appBackupsMap.values()).filter(s => !!s).reduce((acc, cur) => ({
+ fileCount: acc.fileCount + cur.upload.fileCount,
+ size: acc.size + cur.upload.size,
+ transferred: acc.transferred + cur.upload.transferred,
+ startTime: Math.min(acc.startTime, cur.upload.startTime),
+ duration: acc.duration + cur.upload.duration,
+ }), stats.upload);
debug(`backupBox: rotating box snapshot of ${backupSite.id} to id ${remotePath}. ${JSON.stringify(stats)}`);
@@ -306,14 +308,23 @@ async function backupBox(backupSite, appBackupsMap, tag, options, progressCallba
appConfig: null,
siteId: backupSite.id,
stats,
- integrity
+ integrity: uploadResult.integrity
};
const id = await backups.add(data);
const snapshotPath = addFileExtension(backupSite, 'snapshot/box');
+ const copyStartTime = Date.now();
const [error] = await safe(copy(backupSite, snapshotPath, remotePath, progressCallback));
const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
- await backups.setState(id, state);
+ if (!error) {
+ stats.copy = { startTime: copyStartTime, duration: Date.now() - copyStartTime };
+ // stats object might be null for stopped/errored apps from old versions
+ stats.aggregatedCopy = Array.from(appBackupsMap.values()).filter(s => !!s).reduce((acc, cur) => ({
+ startTime: Math.min(acc.startTime, cur.copy.startTime),
+ duration: acc.duration + cur.copy.duration,
+ }), stats.copy);
+ }
+ await backups.update({ id }, { stats, state });
if (error) throw error;
return id;
@@ -378,7 +389,9 @@ async function backupAppWithTag(app, backupSite, tag, options, progressCallback)
return { id: lastKnownGoodAppBackup.id, stats: lastKnownGoodAppBackup.stats };
}
- const { stats, integrity } = await uploadAppSnapshot(backupSite, app, progressCallback);
+ const uploadStartTime = Date.now();
+ const uploadResult = await uploadAppSnapshot(backupSite, app, progressCallback); // { stats, integrity }
+ const stats = { upload: { ...uploadResult.stats, startTime: uploadStartTime, duration: Date.now() - uploadStartTime } };
const manifest = app.manifest;
const remotePath = addFileExtension(backupSite, `${tag}/app_${app.fqdn}_v${manifest.version}`);
@@ -398,17 +411,19 @@ async function backupAppWithTag(app, backupSite, tag, options, progressCallback)
appConfig: app,
siteId: backupSite.id,
stats,
- integrity
+ integrity: uploadResult.integrity
};
const id = await backups.add(data);
const snapshotPath = addFileExtension(backupSite, `snapshot/app_${app.id}`);
+ const copyStartTime = Date.now();
const [error] = await safe(copy(backupSite, snapshotPath, remotePath, progressCallback));
const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
- await backups.setState(id, state);
+ if (!error) stats.copy = { startTime: copyStartTime, duration: Date.now() - copyStartTime };
+ await backups.update({ id }, { stats, state });
if (error) throw error;
- return { id, stats: data.stats };
+ return { id, stats };
}
async function backupApp(app, backupSite, options, progressCallback) {
@@ -467,7 +482,9 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
debug(`backupMailWithTag: backing up mail with tag ${tag}`);
- const { stats, integrity } = await uploadMailSnapshot(backupSite, progressCallback);
+ const uploadStartTime = Date.now();
+ const uploadResult = await uploadMailSnapshot(backupSite, progressCallback); // { stats, integrity }
+ const stats = { upload: { ...uploadResult.stats, startTime: uploadStartTime, duration: Date.now() - uploadStartTime } };
const remotePath = addFileExtension(backupSite, `${tag}/mail_v${constants.VERSION}`);
@@ -486,17 +503,19 @@ async function backupMailWithTag(backupSite, tag, options, progressCallback) {
appConfig: null,
siteId: backupSite.id,
stats,
- integrity
+ integrity: uploadResult.integrity
};
const id = await backups.add(data);
const snapshotPath = addFileExtension(backupSite, 'snapshot/mail');
+ const copyStartTime = Date.now();
const [error] = await safe(copy(backupSite, snapshotPath, remotePath, progressCallback));
const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
- await backups.setState(id, state);
+ if (!error) stats.copy = { startTime: copyStartTime, duration: Date.now() - copyStartTime };
+ await backups.update({ id }, { stats, state });
if (error) throw error;
- return { id, stats: data.stats };
+ return { id, stats };
}
async function downloadMail(backupSite, remotePath, progressCallback) {
diff --git a/src/progress-stream.js b/src/progress-stream.js
index 6b3473628..71b432851 100644
--- a/src/progress-stream.js
+++ b/src/progress-stream.js
@@ -22,8 +22,8 @@ class ProgressStream extends TransformStream {
}
stats() {
- const totalMsecs = Date.now() - this.#startTime;
- return { startTime: this.#startTime, totalMsecs, transferred: this.#transferred };
+ const duration = Date.now() - this.#startTime;
+ return { startTime: this.#startTime, duration, transferred: this.#transferred };
}
_start() {