jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

backuptarget: pseudo target for import and restore

Browse Source
This commit is contained in:
Girish Ramakrishnan
2025-08-01 23:20:51 +02:00
parent 3cabbc1328
commit a01e1bad0f
8 changed files with 90 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/apps.js
View File

@@ -165,7 +165,6 @@ const appTaskManager = require('./apptaskmanager.js'),
domains = require('./domains.js'),
eventlog = require('./eventlog.js'),
fs = require('fs'),
hush = require('./hush.js'),
Location = require('./location.js'),
locks = require('./locks.js'),
logs = require('./logs.js'),
@@ -2350,30 +2349,22 @@ async function importApp(app, data, auditSource) {
const appId = app.id;
const { remotePath, format, encryptionPassword, encryptedFilenames, config } = data;
let error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
const error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
if (error) throw error;
let encryption, restoreConfig;
let restoreConfig;
if (data.remotePath) { // if not provided, we import in-place
error = backupTargets.validateFormat(format);
if (error) throw error;
const backupTarget = await backupTargets.createPseudo({
id: `appimport-${app.id}`,
provider: data.provider,
config: data.config,
format: data.format,
encryptionPassword: data.encryptionPassword,
encryptedFilenames: data.encryptedFilenames
});
if (encryptionPassword) {
encryption = hush.generateEncryptionKeysSync(encryptionPassword);
encryption.encryptedFilenames = !!encryptedFilenames;
} else {
encryption = null;
}
await backupTargets.setupManagedStorage(config, `/mnt/appimport-${app.id}`); // this validates mountOptions . this is not cleaned up, it's fine
config.rootPath = backupTargets.getRootPath(config, `/mnt/appimport-${app.id}`);
error = await backupTargets.testStorage(Object.assign({ mountPath: `/mnt/appimport-${app.id}` }, config)); // this validates provider and it's api options. requires mountPath
if (error) throw error;
restoreConfig = { remotePath, backupFormat: format, backupConfig: config };
restoreConfig = { remotePath: data.remotePath, backupTarget };
} else { // inPlace
restoreConfig = { inPlace: true };
}
@@ -2389,7 +2380,7 @@ async function importApp(app, data, auditSource) {
};
const taskId = await addTask(appId, exports.ISTATE_PENDING_IMPORT, task, auditSource);
await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, remotePath, fromManifest: app.manifest, toManifest: app.manifest, taskId });
await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app, remotePath: data.remotePath, inPlace: data.inPlace, taskId });
return { taskId };
}

9
src/apptask.js
View File

@@ -27,7 +27,6 @@ const apps = require('./apps.js'),
fs = require('fs'),
iputils = require('./iputils.js'),
manifestFormat = require('@cloudron/manifest-format'),
mounts = require('./mounts.js'),
os = require('os'),
path = require('path'),
paths = require('./paths.js'),
@@ -331,13 +330,11 @@ async function installCommand(app, args, progressCallback) {
await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' });
await services.setupAddons(app, app.manifest.addons);
await services.clearAddons(app, app.manifest.addons);
const backupConfig = restoreConfig.backupConfig;
const mountObject = await backupTargets.setupManagedStorage(backupConfig, `/mnt/appimport-${app.id}`);
if (mountObject) await progressCallback({ percent: 70, message: 'Setting up mount for importing' });
backupConfig.rootPath = backupTargets.getRootPath(backupConfig, `/mnt/appimport-${app.id}`);
const backupTarget = restoreConfig.backupTarget;
await backupTargets.storageApi(backupTarget).setup(backupTarget.config);
await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 75, message: progress.message }); });
await apps.loadConfig(app);
if (mountObject) await mounts.removeMount(mountObject);
await backupTargets.storageApi(backupTarget).teardown(backupTarget.config);
await progressCallback({ percent: 75, message: 'Restoring addons' });
await services.restoreAddons(app, app.manifest.addons);
} else { // clone and restore

9
src/backupformat.js
View File

@@ -2,6 +2,7 @@
exports = module.exports = {
api,
validateFormat,
};
const assert = require('assert'),
@@ -17,3 +18,11 @@ function api(format) {
throw new BoxError(BoxError.INTERNAL_ERROR, `Undefined format ${format}`);
}
function validateFormat(format) {
assert.strictEqual(typeof format, 'string');
if (format === 'tgz' || format == 'rsync') return null;
return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
}

39
src/backuptargets.js
View File

@@ -22,8 +22,6 @@ exports = module.exports = {
getSnapshotInfo,
setSnapshotInfo,
validateFormat,
getRootPath,
remount,
@@ -33,9 +31,12 @@ exports = module.exports = {
storageApi,
getBackupFilePath,
createPseudo,
};
const assert = require('assert'),
backupFormat = require('./backupformat.js'),
backups = require('./backups.js'),
BoxError = require('./boxerror.js'),
constants = require('./constants.js'),
@@ -160,14 +161,6 @@ function removePrivateFields(target) {
return target;
}
function validateFormat(format) {
assert.strictEqual(typeof format, 'string');
if (format === 'tgz' || format == 'rsync') return null;
return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
}
function validateLabel(label) {
assert.strictEqual(typeof label, 'string');
@@ -492,7 +485,7 @@ async function add(data, auditSource) {
encryptionPassword = data.encryptionPassword || null,
encryptedFilenames = data.encryptedFilenames || false;
const formatError = validateFormat(format);
const formatError = backupFormat.validateFormat(format);
if (formatError) throw formatError;
const labelError = validateLabel(label);
@@ -522,3 +515,27 @@ async function add(data, auditSource) {
return id;
}
// creates a backup target object that is not in the database
async function createPseudo(data) {
assert.strictEqual(typeof data, 'object');
const { id, provider, config, format } = data; // required
const encryptionPassword = data.encryptionPassword || null,
encryptedFilenames = data.encryptedFilenames || false;
const formatError = backupFormat.validateFormat(format);
if (formatError) throw formatError;
let encryption = null;
if (encryptionPassword) {
const encryptionPasswordError = validateEncryptionPassword(encryptionPassword);
if (encryptionPasswordError) throw encryptionPasswordError;
encryption = hush.generateEncryptionKeysSync(encryptionPassword);
encryption.encryptedFilenames = !!encryptedFilenames;
}
debug('add: validating new storage configuration');
const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config });
return { id, format, provider, config: sanitizedConfig, encryption };
}

13
src/backuptask.js
View File

@@ -99,8 +99,8 @@ async function download(backupTarget, remotePath, dataLayout, progressCallback)
await backupFormat.api(backupTarget.format).download(backupTarget, remotePath, dataLayout, progressCallback);
}
async function restore(backupConfig, remotePath, progressCallback) {
assert.strictEqual(typeof backupConfig, 'object');
async function restore(backupTarget, remotePath, progressCallback) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
@@ -108,7 +108,7 @@ async function restore(backupConfig, remotePath, progressCallback) {
if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
const dataLayout = new DataLayout(boxDataDir, []);
await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);
await download(backupTarget, remotePath, dataLayout, progressCallback);
debug('restore: download completed, importing database');
@@ -463,8 +463,9 @@ async function backupMailWithTag(backupTarget, tag, options, progressCallback) {
return await rotateMailBackup(backupTarget, tag, options, progressCallback);
}
async function downloadMail(restoreConfig, progressCallback) {
assert.strictEqual(typeof restoreConfig, 'object');
async function downloadMail(backupTarget, remotePath, progressCallback) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof progressCallback, 'function');
const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
@@ -473,7 +474,7 @@ async function downloadMail(restoreConfig, progressCallback) {
const startTime = new Date();
await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
await download(backupTarget, remotePath, dataLayout, progressCallback);
debug('downloadMail: time: %s', (new Date() - startTime)/1000);
}

2
src/paths.js
View File

@@ -28,7 +28,7 @@ exports = module.exports = {
DEFAULT_BACKUP_DIR: '/var/backups',
VOLUMES_MOUNT_DIR: '/mnt/volumes',
MANAGED_BACKUP_MOUNT_DIR: '/mnt/cloudronbackup',
MANAGED_BACKUP_MOUNT_DIR: '/mnt/backups',
DOCKER_SOCKET_PATH: '/var/run/docker.sock',
PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),

44
src/provision.js
View File

@@ -20,7 +20,6 @@ const appstore = require('./appstore.js'),
domains = require('./domains.js'),
eventlog = require('./eventlog.js'),
fs = require('fs'),
hush = require('./hush.js'),
mail = require('./mail.js'),
mailServer = require('./mailserver.js'),
network = require('./network.js'),
@@ -172,8 +171,8 @@ async function activate(username, password, email, displayName, ip, auditSource)
};
}
async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, options, auditSource) {
assert.strictEqual(typeof backupConfig, 'object');
async function restoreTask(backupTarget, remotePath, ipv4Config, ipv6Config, options, auditSource) {
assert.strictEqual(typeof backupTarget, 'object');
assert.strictEqual(typeof remotePath, 'string');
assert.strictEqual(typeof ipv4Config, 'object');
assert.strictEqual(typeof ipv6Config, 'object');
@@ -181,15 +180,17 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt
assert.strictEqual(typeof auditSource, 'object');
try {
setProgress('restore', 'Preparing backup target');
await backupTargets.storageApi(backupTarget).setup(backupTarget.config);
setProgress('restore', 'Downloading box backup');
backupConfig.rootPath = backupTargets.getRootPath(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR);
await backuptask.restore(backupConfig, remotePath, (progress) => setProgress('restore', progress.message));
await backuptask.restore(backupTarget, remotePath, (progress) => setProgress('restore', progress.message));
setProgress('restore', 'Downloading mail backup');
const mailBackups = await backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1);
if (mailBackups.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'mail backup not found');
const mailRestoreConfig = { backupConfig, remotePath: mailBackups[0].remotePath, backupFormat: mailBackups[0].format };
await backuptask.downloadMail(mailRestoreConfig, (progress) => setProgress('restore', progress.message));
const mailRemotePath = mailBackups[0].remotePath;
await backuptask.downloadMail(backupTarget, mailRemotePath, (progress) => setProgress('restore', progress.message));
await ensureDhparams();
await network.setIPv4Config(ipv4Config);
@@ -204,8 +205,6 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt
}
await dashboard.setupLocation(location.subdomain, location.domain, auditSource);
delete backupConfig.rootPath;
await backupTargets.setConfig(backupConfig);
await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { remotePath });
setImmediate(() => safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug }));
@@ -236,24 +235,19 @@ async function restore(backupConfig, remotePath, version, ipv4Config, ipv6Config
const activated = await users.isActivated();
if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.');
let error = backupTargets.validateFormat(backupConfig.format);
const backupTarget = await backupTargets.createPseudo({
id: `cloudron-restore`,
provider: backupConfig.provider,
config: backupConfig.config,
format: backupConfig.format,
encryptionPassword: backupConfig.encryptionPassword,
encryptedFilenames: backupConfig.encryptedFilenames
});
const error = await network.testIPv4Config(ipv4Config);
if (error) throw error;
if ('password' in backupConfig) {
backupConfig.encryption = hush.generateEncryptionKeysSync(backupConfig.password);
delete backupConfig.password;
} else {
backupConfig.encryption = null;
}
await backupTargets.setupManagedStorage(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR); // this validates mountOptions
error = await backupTargets.testStorage(Object.assign({ mountPath: paths.MANAGED_BACKUP_MOUNT_DIR }, backupConfig)); // this validates provider and it's api options. requires mountPath
if (error) throw error;
error = await network.testIPv4Config(ipv4Config);
if (error) throw error;
safe(restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background
safe(restoreTask(backupTarget, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background
} catch (error) {
debug('restore: error. %o', error);
gStatus.restore.active = false;

18
src/storage/filesystem.js
View File

@@ -254,10 +254,11 @@ async function setup(config) {
debug('setup: removing old storage configuration');
if (!mounts.isManagedProvider(config.provider)) return;
await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error
const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id);
await safe(mounts.removeMount(mountPath), { debug }); // ignore error
debug('setup: setting up new storage configuration');
await setupManagedMount(config.provider, config.mountOptions, paths.MANAGED_BACKUP_MOUNT_DIR);
await setupManagedMount(config.provider, config.mountOptions, mountPath);
}
async function teardown(config) {
@@ -265,7 +266,8 @@ async function teardown(config) {
if (!mounts.isManagedProvider(config.provider)) return;
await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error
const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id);
await safe(mounts.removeMount(mountPath), { debug }); // ignore error
}
async function verifyConfig({ id, provider, config }) {
@@ -277,7 +279,9 @@ async function verifyConfig({ id, provider, config }) {
if ('chown' in config && typeof config.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
if ('preserveAttributes' in config && typeof config.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
let rootPath, testMountObject;
const managedMountValidationPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, `${id}-validation`);
let rootPath;
if (provider === mounts.MOUNT_TYPE_FILESYSTEM) {
if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
const error = validateDestPath(config.backupFolder);
@@ -296,8 +300,8 @@ async function verifyConfig({ id, provider, config }) {
const error = mounts.validateMountOptions(provider, config.mountOptions);
if (error) throw error;
testMountObject = await setupManagedMount(provider, config.mountOptions, '/mnt/backup-storage-validation'); // this validates mountOptions
rootPath = path.join('/mnt/backup-storage-validation', config.prefix);
await setupManagedMount(provider, config.mountOptions, managedMountValidationPath);
rootPath = path.join(managedMountValidationPath, config.prefix);
} else if (provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
if (!config.mountPoint || typeof config.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
const error = validateDestPath(config.mountPoint);
@@ -324,7 +328,7 @@ async function verifyConfig({ id, provider, config }) {
throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
}
if (testMountObject) await mounts.removeMount('/mnt/backup-storage-validation');
if (mounts.isManagedProvider(provider)) await mounts.removeMount(managedMountValidationPath);
const newConfig = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']);
return { provider, id, ...newConfig };