jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: fix app restore with rsync

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-07-25 13:49:37 +02:00
parent fc4da4408c
commit 5be03c7ab5
1 changed files with 35 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
src/backupformat/rsync.js
View File

@@ -27,14 +27,14 @@ const assert = require('assert'),
stream = require('stream/promises'),
syncer = require('../syncer.js');
function getBackupFilePath(backupConfig, remotePath) {
assert.strictEqual(typeof backupConfig, 'object');
function getBackupFilePath(backupTarget, remotePath) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
// we don't have a rootPath for noop
if (backupConfig.provider === 'noop') return remotePath;
if (backupTarget.provider === 'noop') return remotePath;
return path.join(backupConfig.rootPath, remotePath);
return path.join(backupTarget.config.rootPath, remotePath);
}
async function addFile(sourceFile, encryption, uploader, progressCallback) {
@@ -77,46 +77,46 @@ async function addFile(sourceFile, encryption, uploader, progressCallback) {
await uploader.finish();
}
async function processSyncerChange(change, backupConfig, remotePath, dataLayout, progressCallback) {
async function processSyncerChange(change, backupTarget, remotePath, dataLayout, progressCallback) {
debug('sync: processing task: %j', change);
// the empty task.path is special to signify the directory
const destPath = change.path && backupConfig.encryptedFilenames ? hush.encryptFilePath(change.path, backupConfig.encryption) : change.path;
const backupFilePath = path.join(getBackupFilePath(backupConfig, remotePath), destPath);
const destPath = change.path && backupTarget.encryption?.encryptedFilenames ? hush.encryptFilePath(change.path, backupTarget.encryption) : change.path;
const backupFilePath = path.join(getBackupFilePath(backupTarget, remotePath), destPath);
if (change.operation === 'removedir') {
debug(`Removing directory ${backupFilePath}`);
await storage.api(backupConfig.provider).removeDir(backupConfig, backupFilePath, progressCallback);
await storage.api(backupTarget.provider).removeDir(backupTarget.config, backupFilePath, progressCallback);
} else if (change.operation === 'remove') {
debug(`Removing ${backupFilePath}`);
await storage.api(backupConfig.provider).remove(backupConfig, backupFilePath);
await storage.api(backupTarget.provider).remove(backupTarget.config, backupFilePath);
} else if (change.operation === 'add') {
await promiseRetry({ times: 5, interval: 20000, debug }, async (retryCount) => {
progressCallback({ message: `Adding ${change.path}` + (retryCount > 1 ? ` (Try ${retryCount})` : '') });
debug(`Adding ${change.path} position ${change.position} try ${retryCount}`);
const uploader = await storage.api(backupConfig.provider).upload(backupConfig, backupFilePath);
await addFile(dataLayout.toLocalPath('./' + change.path), backupConfig.encryption, uploader, progressCallback);
const uploader = await storage.api(backupTarget.provider).upload(backupTarget.config, backupFilePath);
await addFile(dataLayout.toLocalPath('./' + change.path), backupTarget.encryption, uploader, progressCallback);
});
}
}
async function sync(backupConfig, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function sync(backupTarget, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
// the number here has to take into account the s3.upload partSize (which is 10MB). So 20=200MB
const concurrency = backupConfig.limits?.syncConcurrency || (backupConfig.provider === 's3' ? 20 : 10);
const concurrency = backupTarget.limits?.syncConcurrency || (backupTarget.provider === 's3' ? 20 : 10);
const changes = await syncer.sync(dataLayout);
debug(`sync: processing ${changes.delQueue.length} deletes and ${changes.addQueue.length} additions`);
const [delError] = await safe(async.eachLimit(changes.delQueue, concurrency, async (change) => await processSyncerChange(change, backupConfig, remotePath, dataLayout, progressCallback)));
const [delError] = await safe(async.eachLimit(changes.delQueue, concurrency, async (change) => await processSyncerChange(change, backupTarget, remotePath, dataLayout, progressCallback)));
debug('sync: done processing deletes. error: %o', delError);
if (delError) throw delError;
const [addError] = await safe(async.eachLimit(changes.addQueue, concurrency, async (change) => await processSyncerChange(change, backupConfig, remotePath, dataLayout, progressCallback)));
const [addError] = await safe(async.eachLimit(changes.addQueue, concurrency, async (change) => await processSyncerChange(change, backupTarget, remotePath, dataLayout, progressCallback)));
debug('sync: done processing adds. error: %o', addError);
if (addError) throw addError;
@@ -191,18 +191,20 @@ async function restoreFsMetadata(dataLayout, metadataFile) {
}
}
async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function downloadDir(backupTarget, backupFilePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof backupFilePath, 'string');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}. encryption filenames: ${backupConfig.encryptedFilenames} content: ${!!backupConfig.encryption}`);
const encryptedFilenames = backupTarget.encryption?.encryptedFilenames || false;
debug(`downloadDir: ${backupFilePath} to ${dataLayout.toString()}. encryption filenames: ${encryptedFilenames} content: ${!!backupTarget.encryption}`);
async function downloadFile(entry) {
let relativePath = path.relative(backupFilePath, entry.fullPath);
if (backupConfig.encryptedFilenames) {
const { error, result } = hush.decryptFilePath(relativePath, backupConfig.encryption);
if (encryptedFilenames) {
const { error, result } = hush.decryptFilePath(relativePath, backupTarget.encryption);
if (error) throw new BoxError(BoxError.CRYPTO_ERROR, 'Unable to decrypt file');
relativePath = result;
}
@@ -212,7 +214,7 @@ async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCal
if (mkdirError) throw new BoxError(BoxError.FS_ERROR, mkdirError.message);
await promiseRetry({ times: 3, interval: 20000 }, async function () {
const [downloadError, sourceStream] = await safe(storage.api(backupConfig.provider).download(backupConfig, entry.fullPath));
const [downloadError, sourceStream] = await safe(storage.api(backupTarget.provider).download(backupTarget.config, entry.fullPath));
if (downloadError) {
progressCallback({ message: `Download ${entry.fullPath} to ${destFilePath} errored: ${downloadError.message}` });
throw downloadError;
@@ -229,8 +231,8 @@ async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCal
const streams = [ sourceStream, ps ];
if (backupConfig.encryption) {
const decryptStream = new DecryptStream(backupConfig.encryption);
if (backupTarget.encryption) {
const decryptStream = new DecryptStream(backupTarget.encryption);
streams.push(decryptStream);
}
@@ -248,36 +250,36 @@ async function downloadDir(backupConfig, backupFilePath, dataLayout, progressCal
}
// https://www.digitalocean.com/community/questions/rate-limiting-on-spaces?answer=40441
const concurrency = backupConfig.limits?.downloadConcurrency || (backupConfig.provider === 's3' ? 30 : 10);
const concurrency = backupTarget.limits?.downloadConcurrency || (backupTarget.provider === 's3' ? 30 : 10);
let marker = null;
while (true) {
const batch = await storage.api(backupConfig.provider).listDir(backupConfig, backupFilePath, marker === null ? 1 : 1000, marker); // try with one file first. if that works out, we continue faster
const batch = await storage.api(backupTarget.provider).listDir(backupTarget.config, backupFilePath, marker === null ? 1 : 1000, marker); // try with one file first. if that works out, we continue faster
await async.eachLimit(batch.entries, concurrency, downloadFile);
if (!batch.marker) break;
marker = batch.marker;
}
}
async function download(backupConfig, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function download(backupTarget, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
const backupFilePath = getBackupFilePath(backupConfig, remotePath);
const backupFilePath = getBackupFilePath(backupTarget, remotePath);
debug(`download: Downloading ${backupFilePath} to ${dataLayout.toString()}`);
await downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback);
await downloadDir(backupTarget, backupFilePath, dataLayout, progressCallback);
await restoreFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
}
async function upload(backupConfig, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function upload(backupTarget, remotePath, dataLayout, progressCallback) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof dataLayout, 'object');
assert.strictEqual(typeof progressCallback, 'function');
await saveFsMetadata(dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`);
await sync(backupConfig, remotePath, dataLayout, progressCallback);
await sync(backupTarget, remotePath, dataLayout, progressCallback);
}