jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backups: add remotePath

Browse Source 
the main motivation is that id can be used in REST API routes. previously,
the id was a path and this had a "/" in it. This made /api/v1/backups/:backupId
not work.
This commit is contained in:
Girish Ramakrishnan
2022-04-04 14:13:27 -07:00
parent 54934c41a7
commit 452a4d9a75
15 changed files with 234 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
src/backuptask.js
View File

@@ -315,9 +315,9 @@ function tarPack(dataLayout, encryption, callback) {
return callback(null, ps);
}
function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof backupId, 'string');
assert.strictEqual(typeof remotePath, 'string');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
@@ -329,7 +329,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
debug('sync: processing task: %j', task);
// the empty task.path is special to signify the directory
const destPath = task.path && backupConfig.encryption ? encryptFilePath(task.path, backupConfig.encryption) : task.path;
const backupFilePath = path.join(storage.getBackupFilePath(backupConfig, backupId, backupConfig.format), destPath);
const backupFilePath = path.join(storage.getBackupFilePath(backupConfig, remotePath, backupConfig.format), destPath);
if (task.operation === 'removedir') {
debug(`Removing directory ${backupFilePath}`);
@@ -341,7 +341,7 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
return storage.api(backupConfig.provider).remove(backupConfig, backupFilePath, iteratorCallback);
}
var retryCount = 0;
let retryCount = 0;
async.retry({ times: 5, interval: 20000 }, function (retryCallback) {
retryCallback = once(retryCallback); // protect again upload() erroring much later after read stream error
@@ -379,7 +379,7 @@ async function saveFsMetadata(dataLayout, metadataFile) {
assert.strictEqual(typeof metadataFile, 'string');
// contains paths prefixed with './'
let metadata = {
const metadata = {
emptyDirs: [],
execFiles: [],
symlinks: []
@@ -407,14 +407,14 @@ async function saveFsMetadata(dataLayout, metadataFile) {
}
// this function is called via backupupload (since it needs root to traverse app's directory)
function upload(backupId, format, dataLayoutString, progressCallback, callback) {
assert.strictEqual(typeof backupId, 'string');
function upload(remotePath, format, dataLayoutString, progressCallback, callback) {
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof format, 'string');
assert.strictEqual(typeof dataLayoutString, 'string');
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
debug(`upload: id ${backupId} format ${format} dataLayout ${dataLayoutString}`);
debug(`upload: path ${remotePath} format ${format} dataLayout ${dataLayoutString}`);
const dataLayout = DataLayout.fromString(dataLayoutString);
@@ -438,13 +438,13 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
});
tarStream.on('error', retryCallback); // already returns BoxError
storage.api(backupConfig.provider).upload(backupConfig, storage.getBackupFilePath(backupConfig, backupId, format), tarStream, retryCallback);
storage.api(backupConfig.provider).upload(backupConfig, storage.getBackupFilePath(backupConfig, remotePath, format), tarStream, retryCallback);
});
}, callback);
} else {
async.series([
saveFsMetadata.bind(null, dataLayout, `${dataLayout.localRoot()}/fsmetadata.json`),
sync.bind(null, backupConfig, backupId, dataLayout, progressCallback)
sync.bind(null, backupConfig, remotePath, dataLayout, progressCallback)
], callback);
}
});
@@ -686,8 +686,8 @@ function runBackupUpload(uploadConfig, progressCallback, callback) {
assert.strictEqual(typeof progressCallback, 'function');
assert.strictEqual(typeof callback, 'function');
const { backupId, backupConfig, dataLayout, progressTag } = uploadConfig;
assert.strictEqual(typeof backupId, 'string');
const { remotePath, backupConfig, dataLayout, progressTag } = uploadConfig;
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof progressTag, 'string');
assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
@@ -702,7 +702,7 @@ function runBackupUpload(uploadConfig, progressCallback, callback) {
envCopy.NODE_OPTIONS = `--max-old-space-size=${heapSize}`;
}
shell.sudo(`backup-${backupId}`, [ BACKUP_UPLOAD_CMD, backupId, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true }, function (error) {
shell.sudo(`backup-${remotePath}`, [ BACKUP_UPLOAD_CMD, remotePath, backupConfig.format, dataLayout.toString() ], { env: envCopy, preserveEnv: true, ipc: true }, function (error) {
if (error && (error.code === null /* signal */ || (error.code !== 0 && error.code !== 50))) { // backuptask crashed
return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Backuptask crashed'));
} else if (error && error.code === 50) { // exited with error
@@ -738,7 +738,7 @@ async function uploadBoxSnapshot(backupConfig, progressCallback) {
if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
const uploadConfig = {
backupId: 'snapshot/box',
remotePath: 'snapshot/box',
backupConfig,
dataLayout: new DataLayout(boxDataDir, []),
progressTag: 'box'
@@ -755,29 +755,22 @@ async function uploadBoxSnapshot(backupConfig, progressCallback) {
await backups.setSnapshotInfo('box', { timestamp: new Date().toISOString(), format: backupConfig.format });
}
async function copy(backupConfig, sourceBackupId, destBackupId, options, progressCallback) {
async function copy(backupConfig, srcRemotePath, destRemotePath, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
assert.strictEqual(typeof sourceBackupId, 'string');
assert.strictEqual(typeof destBackupId, 'string');
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof srcRemotePath, 'string');
assert.strictEqual(typeof destRemotePath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
const format = backupConfig.format;
const { provider, format } = backupConfig;
return new Promise((resolve, reject) => {
const startTime = new Date();
const copyEvents = storage.api(backupConfig.provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, sourceBackupId, format), storage.getBackupFilePath(backupConfig, destBackupId, format));
const copyEvents = storage.api(provider).copy(backupConfig, storage.getBackupFilePath(backupConfig, srcRemotePath, format), storage.getBackupFilePath(backupConfig, destRemotePath, format));
copyEvents.on('progress', (message) => progressCallback({ message }));
copyEvents.on('done', async function (copyBackupError) {
const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
const [error] = await safe(backups.update(destBackupId, { preserveSecs: options.preserveSecs || 0, state }));
if (copyBackupError) return reject(copyBackupError);
copyEvents.on('done', function (error) {
if (error) return reject(error);
debug(`copy: copied successfully to id ${destBackupId}. Took ${(new Date() - startTime)/1000} seconds`);
debug(`copy: copied successfully to ${destRemotePath}. Took ${(new Date() - startTime)/1000} seconds`);
resolve();
});
});
@@ -790,12 +783,13 @@ async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCa
assert(Array.isArray(dependsOn));
assert.strictEqual(typeof progressCallback, 'function');
const backupId = `${tag}/box_v${constants.VERSION}`;
const remotePath = `${tag}/box_v${constants.VERSION}`;
const format = backupConfig.format;
debug(`rotateBoxBackup: rotating to id ${backupId}`);
debug(`rotateBoxBackup: rotating to id ${remotePath}`);
const data = {
remotePath,
encryptionVersion: backupConfig.encryption ? 2 : null,
packageVersion: constants.VERSION,
type: backups.BACKUP_TYPE_BOX,
@@ -806,9 +800,13 @@ async function rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCa
format
};
await backups.add(backupId, data);
await copy(backupConfig, 'snapshot/box', backupId, options, progressCallback);
return backupId;
const id = await backups.add(data);
const [copyBackupError] = await safe(copy(backupConfig, 'snapshot/box', remotePath, progressCallback));
const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
await backups.update(id, { preserveSecs: options.preserveSecs || 0, state });
if (copyBackupError) throw copyBackupError;
return id;
}
async function backupBox(dependsOn, tag, options, progressCallback) {
@@ -820,9 +818,7 @@ async function backupBox(dependsOn, tag, options, progressCallback) {
const backupConfig = await settings.getBackupConfig();
await uploadBoxSnapshot(backupConfig, progressCallback);
const backupId = await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback);
return backupId;
return await rotateBoxBackup(backupConfig, tag, options, dependsOn, progressCallback);
}
async function rotateAppBackup(backupConfig, app, tag, options, progressCallback) {
@@ -835,12 +831,13 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback
const snapshotInfo = backups.getSnapshotInfo(app.id);
const manifest = snapshotInfo.restoreConfig ? snapshotInfo.restoreConfig.manifest : snapshotInfo.manifest; // compat
const backupId = `${tag}/app_${app.fqdn}_v${manifest.version}`;
const remotePath = `${tag}/app_${app.fqdn}_v${manifest.version}`;
const format = backupConfig.format;
debug(`rotateAppBackup: rotating ${app.fqdn} to id ${backupId}`);
debug(`rotateAppBackup: rotating ${app.fqdn} to path ${remotePath}`);
const data = {
remotePath,
encryptionVersion: backupConfig.encryption ? 2 : null,
packageVersion: manifest.version,
type: backups.BACKUP_TYPE_APP,
@@ -851,9 +848,13 @@ async function rotateAppBackup(backupConfig, app, tag, options, progressCallback
format: format
};
await backups.add(backupId, data);
await copy(backupConfig, `snapshot/app_${app.id}`, backupId, options, progressCallback);
return backupId;
const id = await backups.add(data);
const copyBackupError = await safe(copy(backupConfig, `snapshot/app_${app.id}`, remotePath, progressCallback));
const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
await backups.update(id, { preserveSecs: options.preserveSecs || 0, state });
if (copyBackupError) throw copyBackupError;
return id;
}
async function backupApp(app, options, progressCallback) {
@@ -890,7 +891,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) {
await snapshotApp(app, progressCallback);
const backupId = util.format('snapshot/app_%s', app.id);
const remotePath = util.format('snapshot/app_%s', app.id);
const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id));
if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`);
@@ -899,7 +900,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) {
progressCallback({ message: `Uploading app snapshot ${app.fqdn}`});
const uploadConfig = {
backupId,
remotePath,
backupConfig,
dataLayout,
progressTag: app.fqdn
@@ -909,7 +910,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) {
await util.promisify(runBackupUpload)(uploadConfig, progressCallback);
debug(`uploadAppSnapshot: ${app.fqdn} upload with id ${backupId}. ${(new Date() - startTime)/1000} seconds`);
debug(`uploadAppSnapshot: ${app.fqdn} upload to ${remotePath}. ${(new Date() - startTime)/1000} seconds`);
await backups.setSnapshotInfo(app.id, { timestamp: new Date().toISOString(), manifest: app.manifest, format: backupConfig.format });
}
@@ -930,8 +931,7 @@ async function backupAppWithTag(app, tag, options, progressCallback) {
const backupConfig = await settings.getBackupConfig();
await uploadAppSnapshot(backupConfig, app, progressCallback);
const backupId = await rotateAppBackup(backupConfig, app, tag, options, progressCallback);
return backupId;
return await rotateAppBackup(backupConfig, app, tag, options, progressCallback);
}
async function uploadMailSnapshot(backupConfig, progressCallback) {
@@ -942,7 +942,7 @@ async function uploadMailSnapshot(backupConfig, progressCallback) {
if (!mailDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving maildata: ${safe.error.message}`);
const uploadConfig = {
backupId: 'snapshot/mail',
remotePath: 'snapshot/mail',
backupConfig,
dataLayout: new DataLayout(mailDataDir, []),
progressTag: 'mail'
@@ -965,12 +965,13 @@ async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const backupId = `${tag}/mail_v${constants.VERSION}`;
const remotePath = `${tag}/mail_v${constants.VERSION}`;
const format = backupConfig.format;
debug(`rotateMailBackup: rotating to id ${backupId}`);
debug(`rotateMailBackup: rotating to ${remotePath}`);
const data = {
remotePath,
encryptionVersion: backupConfig.encryption ? 2 : null,
packageVersion: constants.VERSION,
type: backups.BACKUP_TYPE_MAIL,
@@ -981,9 +982,13 @@ async function rotateMailBackup(backupConfig, tag, options, progressCallback) {
format: format
};
await backups.add(backupId, data);
await copy(backupConfig, 'snapshot/mail', backupId, options, progressCallback);
return backupId;
const id = await backups.add(data);
const [copyBackupError] = await safe(copy(backupConfig, 'snapshot/mail', remotePath, progressCallback));
const state = copyBackupError ? backups.BACKUP_STATE_ERROR : backups.BACKUP_STATE_NORMAL;
await backups.update(id, { preserveSecs: options.preserveSecs || 0, state });
if (copyBackupError) throw copyBackupError;
return id;
}
async function backupMailWithTag(tag, options, progressCallback) {
@@ -995,8 +1000,7 @@ async function backupMailWithTag(tag, options, progressCallback) {
const backupConfig = await settings.getBackupConfig();
await uploadMailSnapshot(backupConfig, progressCallback);
const backupId = await rotateMailBackup(backupConfig, tag, options, progressCallback);
return backupId;
return await rotateMailBackup(backupConfig, tag, options, progressCallback);
}
async function backupMail(options, progressCallback) {
@@ -1028,7 +1032,7 @@ async function fullBackup(options, progressCallback) {
assert.strictEqual(typeof options, 'object');
assert.strictEqual(typeof progressCallback, 'function');
const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,'');
const tag = (new Date()).toISOString().replace(/[T.]/g, '-').replace(/[:Z]/g,''); // unique tag under which all apps/mail/box backs up
const allApps = await apps.list();
@@ -1047,19 +1051,19 @@ async function fullBackup(options, progressCallback) {
}
const startTime = new Date();
const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
const appBackupId = await backupAppWithTag(app, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
debug(`fullBackup: app ${app.fqdn} backup finished. Took ${(new Date() - startTime)/1000} seconds`);
if (appBackupId) appBackupIds.push(appBackupId); // backupId can be null if in BAD_STATE and never backed up
}
progressCallback({ percent: percent, message: 'Backing up mail' });
progressCallback({ percent, message: 'Backing up mail' });
percent += step;
const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
const mailBackupId = await backupMailWithTag(tag, options, (progress) => progressCallback({ percent, message: progress.message }));
progressCallback({ percent: percent, message: 'Backing up system data' });
progressCallback({ percent, message: 'Backing up system data' });
percent += step;
const dependsOn = appBackupIds.concat(mailBackupId);
const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent: percent, message: progress.message }));
const backupId = await backupBox(dependsOn, tag, options, (progress) => progressCallback({ percent, message: progress.message }));
return backupId;
}