diff --git a/migrations/20250812141107-backups-add-statsJson.js b/migrations/20250812141107-backups-add-statsJson.js new file mode 100644 index 000000000..f2290e0cb --- /dev/null +++ b/migrations/20250812141107-backups-add-statsJson.js @@ -0,0 +1,15 @@ +'use strict'; + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE backups ADD COLUMN statsJson TEXT', function (error) { + if (error) console.error(error); + callback(error); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE backups DROP COLUMN statsJson', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/src/backupformat/tgz.js b/src/backupformat/tgz.js index 7c9bbc031..7f91afd74 100644 --- a/src/backupformat/tgz.js +++ b/src/backupformat/tgz.js @@ -2,7 +2,6 @@ const assert = require('assert'), backupTargets = require('../backuptargets.js'), - blobs = require('../blobs.js'), BoxError = require('../boxerror.js'), crypto = require('crypto'), DataLayout = require('../datalayout.js'), @@ -164,7 +163,10 @@ async function tarPack(dataLayout, encryption, uploader, progressCallback) { debug(`tarPack: pipeline finished: ${JSON.stringify(ps.stats())}`); await uploader.finish(); - return { size: ps.stats().transferred, sha256: hash.digest('hex') }; + return { + stats: ps.stats(), + integrity: { size: ps.stats().transferred, sha256: hash.digest('hex') } + }; } async function tarExtract(inStream, dataLayout, encryption, progressCallback) { @@ -252,17 +254,17 @@ async function upload(backupTarget, remotePath, dataLayout, progressCallback) { progressCallback({ message: `Uploading backup ${remotePath}` }); const uploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, remotePath); - const { size, sha256 } = await tarPack(dataLayout, backupTarget.encryption, uploader, progressCallback); + const { stats, integrity } = await tarPack(dataLayout, backupTarget.encryption, uploader, progressCallback); - const checksumData = [{ filename: path.basename(remotePath), size, sha256 }]; + const checksumData = [{ filename: path.basename(remotePath), ...integrity }]; const checksumDataJsonString = JSON.stringify(checksumData, null, 4); const checksumDataStream = Readable.from(checksumDataJsonString); const checksumUploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, `${remotePath}.checksum`); await stream.pipeline(checksumDataStream, checksumUploader.stream); await checksumUploader.finish(); - const checksumSignature = await crypto.sign(null /* algorithm */, checksumDataJsonString, backupTarget.integrityKeyPair.privateKey); - return { size, sha256, checksumSignature }; + integrity.signature = await crypto.sign(null /* algorithm */, checksumDataJsonString, backupTarget.integrityKeyPair.privateKey); + return { stats, integrity }; }); } diff --git a/src/backups.js b/src/backups.js index 90605e3a3..5f83b2f23 100644 --- a/src/backups.js +++ b/src/backups.js @@ -8,7 +8,6 @@ exports = module.exports = { add, update, setState, - setIntegrity, list, del, @@ -30,7 +29,7 @@ const assert = require('assert'), hat = require('./hat.js'), safe = require('safetydance'); -const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'integrityJson', 'dependsOnJson', 'state', 'manifestJson', 'preserveSecs', 'encryptionVersion', 'appConfigJson', 'targetId' ].join(','); +const BACKUPS_FIELDS = [ 'id', 'remotePath', 'label', 'identifier', 'creationTime', 'packageVersion', 'type', 'integrityJson', 'statsJson', 'dependsOnJson', 'state', 'manifestJson', 'preserveSecs', 'encryptionVersion', 'appConfigJson', 'targetId' ].join(','); function postProcess(result) { assert.strictEqual(typeof result, 'object'); @@ -44,6 +43,9 @@ function postProcess(result) { result.integrity = result.integrityJson ? safe.JSON.parse(result.integrityJson) : null; delete result.integrityJson; + result.stats = result.statsJson ? safe.JSON.parse(result.statsJson) : null; + delete result.statsJson; + result.appConfig = result.appConfigJson ? safe.JSON.parse(result.appConfigJson) : null; delete result.appConfigJson; @@ -69,9 +71,11 @@ async function add(data) { const prefixId = data.type === exports.BACKUP_TYPE_APP ? `${data.type}_${data.identifier}` : data.type; // type and identifier are same for other types const id = `${prefixId}_v${data.packageVersion}_${hat(32)}`; // id is used by the UI to derive dependent packages. making this a UUID will require a lot of db querying const appConfigJson = data.appConfig ? JSON.stringify(data.appConfig) : null; + const statsJson = data.statsJson ? JSON.stringify(data.statsJson) : null; + const integrityJson = data.integrityJson ? JSON.stringify(data.integrityJson) : null; - const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, preserveSecs, appConfigJson, targetId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', - [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.preserveSecs, appConfigJson, data.targetId ])); + const [error] = await safe(database.query('INSERT INTO backups (id, remotePath, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOnJson, manifestJson, preserveSecs, appConfigJson, targetId, statsJson, integrityJson) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + [ id, data.remotePath, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, JSON.stringify(data.dependsOn), manifestJson, data.preserveSecs, appConfigJson, data.targetId, statsJson, integrityJson ])); if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'Backup already exists'); if (error) throw error; @@ -170,14 +174,6 @@ async function setState(id, state) { if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); } -async function setIntegrity(id, integrity) { - assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof integrity, 'object'); - - const result = await database.query('UPDATE backups SET integrityJson = ? WHERE id = ?', [JSON.stringify(integrity), id]); - if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Backup not found'); -} - async function list(page, perPage) { assert(typeof page === 'number' && page > 0); assert(typeof perPage === 'number' && perPage > 0); diff --git a/src/backuptask.js b/src/backuptask.js index 15f82b7d3..43edf281c 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -210,13 +210,13 @@ async function uploadBoxSnapshot(backupTarget, progressCallback) { const startTime = new Date(); - const integrity = await runBackupUpload(uploadConfig, progressCallback); + const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); debug(`uploadBoxSnapshot: took ${(new Date() - startTime)/1000} seconds`); await backupTargets.setSnapshotInfo(backupTarget, 'box', { timestamp: new Date().toISOString() }); - return integrity; + return { stats, integrity }; } async function copy(backupTarget, srcRemotePath, destRemotePath, progressCallback) { @@ -241,16 +241,18 @@ async function copy(backupTarget, srcRemotePath, destRemotePath, progressCallbac debug(`copy: copied checksum successfully to ${destRemotePath}.checksum`); } -async function rotateBoxBackup(backupTarget, tag, options, dependsOn, progressCallback) { +async function backupBox(backupTarget, dependsOn, tag, options, progressCallback) { assert.strictEqual(typeof backupTarget, 'object'); + assert(Array.isArray(dependsOn)); assert.strictEqual(typeof tag, 'string'); assert.strictEqual(typeof options, 'object'); - assert(Array.isArray(dependsOn)); assert.strictEqual(typeof progressCallback, 'function'); + const { stats, integrity } = await uploadBoxSnapshot(backupTarget, progressCallback); + const remotePath = addFileExtension(backupTarget, `${tag}/box_v${constants.VERSION}`); - debug(`rotateBoxBackup: rotating to id ${remotePath}`); + debug(`backupBox: rotating to id ${remotePath}`); const data = { remotePath, @@ -263,7 +265,9 @@ async function rotateBoxBackup(backupTarget, tag, options, dependsOn, progressCa manifest: null, preserveSecs: options.preserveSecs || 0, appConfig: null, - targetId: backupTarget.id + targetId: backupTarget.id, + stats, + integrity }; const id = await backups.add(data); @@ -276,74 +280,6 @@ async function rotateBoxBackup(backupTarget, tag, options, dependsOn, progressCa return id; } -async function backupBox(backupTarget, dependsOn, tag, options, progressCallback) { - assert.strictEqual(typeof backupTarget, 'object'); - assert(Array.isArray(dependsOn)); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - - const integrity = await uploadBoxSnapshot(backupTarget, progressCallback); - const id = await rotateBoxBackup(backupTarget, tag, options, dependsOn, progressCallback); - await backups.setIntegrity(id, integrity); - return id; -} - -async function rotateAppBackup(backupTarget, app, tag, options, progressCallback) { - assert.strictEqual(typeof backupTarget, 'object'); - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - - const manifest = app.manifest; - const remotePath = addFileExtension(backupTarget, `${tag}/app_${app.fqdn}_v${manifest.version}`); - - debug(`rotateAppBackup: rotating ${app.fqdn} to path ${remotePath}`); - - const data = { - remotePath, - encryptionVersion: backupTarget.encryption ? 2 : null, - packageVersion: manifest.version, - type: backups.BACKUP_TYPE_APP, - state: backups.BACKUP_STATE_CREATING, - identifier: app.id, - dependsOn: [], - manifest, - preserveSecs: options.preserveSecs || 0, - appConfig: app, - targetId: backupTarget.id - }; - - const id = await backups.add(data); - const snapshotPath = addFileExtension(backupTarget, `snapshot/app_${app.id}`); - const [error] = await safe(copy(backupTarget, snapshotPath, remotePath, progressCallback)); - const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; - await backups.setState(id, state); - if (error) throw error; - - return id; -} - -async function backupApp(app, backupTarget, options, progressCallback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof backupTarget, 'object'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - - let backupId = null; - await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); - if (options.snapshotOnly) { - await snapshotApp(app, progressCallback); - } else { - const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); - backupId = await backupAppWithTag(app, backupTarget, tag, options, progressCallback); - } - await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); - - return backupId; -} - async function snapshotApp(app, progressCallback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof progressCallback, 'function'); @@ -381,13 +317,13 @@ async function uploadAppSnapshot(backupTarget, app, progressCallback) { const startTime = new Date(); - const integrity = await runBackupUpload(uploadConfig, progressCallback); + const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); debug(`uploadAppSnapshot: ${app.fqdn} uploaded to ${remotePath}. ${(new Date() - startTime)/1000} seconds`); await backupTargets.setSnapshotInfo(backupTarget, app.id, { timestamp: new Date().toISOString(), manifest: app.manifest }); - return integrity; + return { stats, integrity }; } async function backupAppWithTag(app, backupTarget, tag, options, progressCallback) { @@ -403,12 +339,58 @@ async function backupAppWithTag(app, backupTarget, tag, options, progressCallbac return lastKnownGoodAppBackup.id; } - const integrity = await uploadAppSnapshot(backupTarget, app, progressCallback); - const id = await rotateAppBackup(backupTarget, app, tag, options, progressCallback); - await backups.setIntegrity(id, integrity); + const { stats, integrity } = await uploadAppSnapshot(backupTarget, app, progressCallback); + + const manifest = app.manifest; + const remotePath = addFileExtension(backupTarget, `${tag}/app_${app.fqdn}_v${manifest.version}`); + + debug(`backupAppWithTag: rotating ${app.fqdn} to path ${remotePath}`); + + const data = { + remotePath, + encryptionVersion: backupTarget.encryption ? 2 : null, + packageVersion: manifest.version, + type: backups.BACKUP_TYPE_APP, + state: backups.BACKUP_STATE_CREATING, + identifier: app.id, + dependsOn: [], + manifest, + preserveSecs: options.preserveSecs || 0, + appConfig: app, + targetId: backupTarget.id, + stats, + integrity + }; + + const id = await backups.add(data); + const snapshotPath = addFileExtension(backupTarget, `snapshot/app_${app.id}`); + const [error] = await safe(copy(backupTarget, snapshotPath, remotePath, progressCallback)); + const state = error ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL; + await backups.setState(id, state); + if (error) throw error; + return id; } +async function backupApp(app, backupTarget, options, progressCallback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof backupTarget, 'object'); + assert.strictEqual(typeof options, 'object'); + assert.strictEqual(typeof progressCallback, 'function'); + + let backupId = null; + await locks.wait(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); + if (options.snapshotOnly) { + await snapshotApp(app, progressCallback); + } else { + const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); + backupId = await backupAppWithTag(app, backupTarget, tag, options, progressCallback); + } + await locks.release(`${locks.TYPE_APP_BACKUP_PREFIX}${app.id}`); + + return backupId; +} + async function uploadMailSnapshot(backupTarget, progressCallback) { assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof progressCallback, 'function'); @@ -429,24 +411,28 @@ async function uploadMailSnapshot(backupTarget, progressCallback) { const startTime = new Date(); - const integrity = await runBackupUpload(uploadConfig, progressCallback); + const { stats, integrity } = await runBackupUpload(uploadConfig, progressCallback); debug(`uploadMailSnapshot: took ${(new Date() - startTime)/1000} seconds`); await backupTargets.setSnapshotInfo(backupTarget, 'mail', { timestamp: new Date().toISOString() }); - return integrity; + return { stats, integrity }; } -async function rotateMailBackup(backupTarget, tag, options, progressCallback) { +async function backupMailWithTag(backupTarget, tag, options, progressCallback) { assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof tag, 'string'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof progressCallback, 'function'); + debug(`backupMailWithTag: backing up mail with tag ${tag}`); + + const { stats, integrity } = await uploadMailSnapshot(backupTarget, progressCallback); + const remotePath = addFileExtension(backupTarget, `${tag}/mail_v${constants.VERSION}`); - debug(`rotateMailBackup: rotating to ${remotePath}`); + debug(`backupMailWithTag: rotating to ${remotePath}`); const data = { remotePath, @@ -459,7 +445,9 @@ async function rotateMailBackup(backupTarget, tag, options, progressCallback) { manifest: null, preserveSecs: options.preserveSecs || 0, appConfig: null, - targetId: backupTarget.id + targetId: backupTarget.id, + stats, + integrity }; const id = await backups.add(data); @@ -472,20 +460,6 @@ async function rotateMailBackup(backupTarget, tag, options, progressCallback) { return id; } -async function backupMailWithTag(backupTarget, tag, options, progressCallback) { - assert.strictEqual(typeof backupTarget, 'object'); - assert.strictEqual(typeof tag, 'string'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof progressCallback, 'function'); - - debug(`backupMailWithTag: backing up mail with tag ${tag}`); - - const integrity = await uploadMailSnapshot(backupTarget, progressCallback); - const id = await rotateMailBackup(backupTarget, tag, options, progressCallback); - await backups.setIntegrity(id, integrity); - return id; -} - async function downloadMail(backupTarget, remotePath, progressCallback) { assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); diff --git a/src/test/backups-test.js b/src/test/backups-test.js index 0962202b6..5f23be806 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -29,7 +29,8 @@ describe('backups', function () { label: '', appConfig: null, targetId: null, - integrity: null + integrity: null, + stats: null }; const appBackup = { @@ -46,7 +47,8 @@ describe('backups', function () { label: '', appConfig: null, targetId: null, - integrity: null + integrity: null, + stats: null }; let defaultBackupTarget;