diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js index a2d28a2b2..f08e005e2 100644 --- a/src/backupformat/rsync.js +++ b/src/backupformat/rsync.js @@ -13,7 +13,6 @@ const assert = require('node:assert'), async = require('async'), backupTargets = require('../backuptargets.js'), BoxError = require('../boxerror.js'), - crypto = require('node:crypto'), DataLayout = require('../datalayout.js'), { DecryptStream } = require('../hush.js'), debug = require('debug')('box:backupformat/rsync'), @@ -25,7 +24,6 @@ const assert = require('node:assert'), paths = require('../paths.js'), ProgressStream = require('../progress-stream.js'), promiseRetry = require('../promise-retry.js'), - { Readable } = require('node:stream'), safe = require('safetydance'), shell = require('../shell.js')('backupformat/rsync'), stream = require('stream/promises'), @@ -129,7 +127,7 @@ async function sync(backupTarget, remotePath, dataLayout, progressCallback) { return { stats: { ...aggregatedStats, totalMsecs: Date.now()-aggregatedStats.startTime }, - integrity: [...integrityMap.entries()].sort(([a], [b]) => a < b) // for readability, order the entries + integrityMap }; } @@ -289,15 +287,7 @@ async function upload(backupTarget, remotePath, dataLayout, progressCallback) { assert.strictEqual(typeof progressCallback, 'function'); await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`); - const { stats, integrity } = await sync(backupTarget, remotePath, dataLayout, progressCallback); - const integrityDataJsonString = JSON.stringify(Object.fromEntries(integrity), null, 2); - const integrityDataStream = Readable.from(integrityDataJsonString); - const integrityUploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, `${remotePath}.backupinfo`); - await stream.pipeline(integrityDataStream, integrityUploader.stream); - await integrityUploader.finish(); - - const signature = await crypto.sign(null /* algorithm */, integrityDataJsonString, backupTarget.integrityKeyPair.privateKey); - return { stats, integrity: { signature } }; + return await sync(backupTarget, remotePath, dataLayout, progressCallback); // { stats, integrityMap } } function getFileExtension(encryption) { diff --git a/src/backupformat/tgz.js b/src/backupformat/tgz.js index af48ac6a4..8f1457f40 100644 --- a/src/backupformat/tgz.js +++ b/src/backupformat/tgz.js @@ -3,7 +3,6 @@ const assert = require('node:assert'), backupTargets = require('../backuptargets.js'), BoxError = require('../boxerror.js'), - crypto = require('node:crypto'), DataLayout = require('../datalayout.js'), debug = require('debug')('box:backupformat/tgz'), { DecryptStream, EncryptStream } = require('../hush.js'), @@ -12,7 +11,6 @@ const assert = require('node:assert'), path = require('node:path'), ProgressStream = require('../progress-stream.js'), promiseRetry = require('../promise-retry.js'), - { Readable } = require('node:stream'), safe = require('safetydance'), stream = require('stream/promises'), { Transform } = require('node:stream'), @@ -266,16 +264,8 @@ async function upload(backupTarget, remotePath, dataLayout, progressCallback) { const uploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, remotePath); const { stats, integrity } = await tarPack(dataLayout, backupTarget.encryption, uploader, progressCallback); - const integrityMap = new Map(); - integrityMap.set(path.basename(remotePath), integrity); - const integrityDataJsonString = JSON.stringify(Object.fromEntries(integrityMap), null, 2); - const integrityDataStream = Readable.from(integrityDataJsonString); - const integrityUploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, `${remotePath}.backupinfo`); - await stream.pipeline(integrityDataStream, integrityUploader.stream); - await integrityUploader.finish(); - - const signature = await crypto.sign(null /* algorithm */, integrityDataJsonString, backupTarget.integrityKeyPair.privateKey); - return { stats, integrity: { signature } }; + const integrityMap = new Map([ [path.basename(remotePath), integrity] ]); + return { stats, integrityMap }; }); } diff --git a/src/backuptask.js b/src/backuptask.js index 40b05166b..aca2e5edf 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -28,9 +28,11 @@ const apps = require('./apps.js'), locks = require('./locks.js'), path = require('node:path'), paths = require('./paths.js'), + { Readable } = require('node:stream'), safe = require('safetydance'), services = require('./services.js'), - shell = require('./shell.js')('backuptask'); + shell = require('./shell.js')('backuptask'), + stream = require('stream/promises'); const BACKUP_UPLOAD_CMD = path.join(__dirname, 'scripts/backupupload.js'); @@ -69,6 +71,17 @@ async function checkPreconditions(backupTarget, dataLayout) { if (available <= needed) throw new BoxError(BoxError.FS_ERROR, `Not enough disk space for backup. Needed: ${df.prettyBytes(needed)} Available: ${df.prettyBytes(available)}`); } +async function uploadBackupInfo(backupTarget, remotePath, integrityMap) { + const sortedIntegrityMap = [...integrityMap.entries()].sort(([a], [b]) => a < b); // for readability, order the entries + const integrityDataJsonString = JSON.stringify(Object.fromEntries(sortedIntegrityMap), null, 2); + const integrityDataStream = Readable.from(integrityDataJsonString); + const integrityUploader = await backupTargets.storageApi(backupTarget).upload(backupTarget.config, `${remotePath}.backupinfo`); + await stream.pipeline(integrityDataStream, integrityUploader.stream); + await integrityUploader.finish(); + + return await crypto.sign(null /* algorithm */, integrityDataJsonString, backupTarget.integrityKeyPair.privateKey); +} + // this function is called via backupupload (since it needs root to traverse app's directory) async function upload(remotePath, targetId, dataLayoutString, progressCallback) { assert.strictEqual(typeof remotePath, 'string'); @@ -85,7 +98,10 @@ async function upload(remotePath, targetId, dataLayoutString, progressCallback) await checkPreconditions(backupTarget, dataLayout); - return await backupFormat.api(backupTarget.format).upload(backupTarget, remotePath, dataLayout, progressCallback); + const { stats, integrityMap } = await backupFormat.api(backupTarget.format).upload(backupTarget, remotePath, dataLayout, progressCallback); + + const signature = await uploadBackupInfo(backupTarget, remotePath, integrityMap); + return { stats, integrity: { signature } }; } async function download(backupTarget, remotePath, dataLayout, progressCallback) {