diff --git a/src/apps.js b/src/apps.js index 088543e54..3c33689ce 100644 --- a/src/apps.js +++ b/src/apps.js @@ -165,7 +165,6 @@ const appTaskManager = require('./apptaskmanager.js'), domains = require('./domains.js'), eventlog = require('./eventlog.js'), fs = require('fs'), - hush = require('./hush.js'), Location = require('./location.js'), locks = require('./locks.js'), logs = require('./logs.js'), @@ -2350,30 +2349,22 @@ async function importApp(app, data, auditSource) { const appId = app.id; - const { remotePath, format, encryptionPassword, encryptedFilenames, config } = data; - - let error = checkAppState(app, exports.ISTATE_PENDING_IMPORT); + const error = checkAppState(app, exports.ISTATE_PENDING_IMPORT); if (error) throw error; - let encryption, restoreConfig; + let restoreConfig; if (data.remotePath) { // if not provided, we import in-place - error = backupTargets.validateFormat(format); - if (error) throw error; + const backupTarget = await backupTargets.createPseudo({ + id: `appimport-${app.id}`, + provider: data.provider, + config: data.config, + format: data.format, + encryptionPassword: data.encryptionPassword, + encryptedFilenames: data.encryptedFilenames + }); - if (encryptionPassword) { - encryption = hush.generateEncryptionKeysSync(encryptionPassword); - encryption.encryptedFilenames = !!encryptedFilenames; - } else { - encryption = null; - } - - await backupTargets.setupManagedStorage(config, `/mnt/appimport-${app.id}`); // this validates mountOptions . this is not cleaned up, it's fine - config.rootPath = backupTargets.getRootPath(config, `/mnt/appimport-${app.id}`); - error = await backupTargets.testStorage(Object.assign({ mountPath: `/mnt/appimport-${app.id}` }, config)); // this validates provider and it's api options. requires mountPath - if (error) throw error; - - restoreConfig = { remotePath, backupFormat: format, backupConfig: config }; + restoreConfig = { remotePath: data.remotePath, backupTarget }; } else { // inPlace restoreConfig = { inPlace: true }; } @@ -2389,7 +2380,7 @@ async function importApp(app, data, auditSource) { }; const taskId = await addTask(appId, exports.ISTATE_PENDING_IMPORT, task, auditSource); - await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, remotePath, fromManifest: app.manifest, toManifest: app.manifest, taskId }); + await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app, remotePath: data.remotePath, inPlace: data.inPlace, taskId }); return { taskId }; } diff --git a/src/apptask.js b/src/apptask.js index 70955a392..cedccf486 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -27,7 +27,6 @@ const apps = require('./apps.js'), fs = require('fs'), iputils = require('./iputils.js'), manifestFormat = require('@cloudron/manifest-format'), - mounts = require('./mounts.js'), os = require('os'), path = require('path'), paths = require('./paths.js'), @@ -331,13 +330,11 @@ async function installCommand(app, args, progressCallback) { await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' }); await services.setupAddons(app, app.manifest.addons); await services.clearAddons(app, app.manifest.addons); - const backupConfig = restoreConfig.backupConfig; - const mountObject = await backupTargets.setupManagedStorage(backupConfig, `/mnt/appimport-${app.id}`); - if (mountObject) await progressCallback({ percent: 70, message: 'Setting up mount for importing' }); - backupConfig.rootPath = backupTargets.getRootPath(backupConfig, `/mnt/appimport-${app.id}`); + const backupTarget = restoreConfig.backupTarget; + await backupTargets.storageApi(backupTarget).setup(backupTarget.config); await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 75, message: progress.message }); }); await apps.loadConfig(app); - if (mountObject) await mounts.removeMount(mountObject); + await backupTargets.storageApi(backupTarget).teardown(backupTarget.config); await progressCallback({ percent: 75, message: 'Restoring addons' }); await services.restoreAddons(app, app.manifest.addons); } else { // clone and restore diff --git a/src/backupformat.js b/src/backupformat.js index b7a163790..ff5a429aa 100644 --- a/src/backupformat.js +++ b/src/backupformat.js @@ -2,6 +2,7 @@ exports = module.exports = { api, + validateFormat, }; const assert = require('assert'), @@ -17,3 +18,11 @@ function api(format) { throw new BoxError(BoxError.INTERNAL_ERROR, `Undefined format ${format}`); } + +function validateFormat(format) { + assert.strictEqual(typeof format, 'string'); + + if (format === 'tgz' || format == 'rsync') return null; + + return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format'); +} diff --git a/src/backuptargets.js b/src/backuptargets.js index db06b653d..4448d19f7 100644 --- a/src/backuptargets.js +++ b/src/backuptargets.js @@ -22,8 +22,6 @@ exports = module.exports = { getSnapshotInfo, setSnapshotInfo, - validateFormat, - getRootPath, remount, @@ -33,9 +31,12 @@ exports = module.exports = { storageApi, getBackupFilePath, + + createPseudo, }; const assert = require('assert'), + backupFormat = require('./backupformat.js'), backups = require('./backups.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), @@ -160,14 +161,6 @@ function removePrivateFields(target) { return target; } -function validateFormat(format) { - assert.strictEqual(typeof format, 'string'); - - if (format === 'tgz' || format == 'rsync') return null; - - return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format'); -} - function validateLabel(label) { assert.strictEqual(typeof label, 'string'); @@ -492,7 +485,7 @@ async function add(data, auditSource) { encryptionPassword = data.encryptionPassword || null, encryptedFilenames = data.encryptedFilenames || false; - const formatError = validateFormat(format); + const formatError = backupFormat.validateFormat(format); if (formatError) throw formatError; const labelError = validateLabel(label); @@ -522,3 +515,27 @@ async function add(data, auditSource) { return id; } + +// creates a backup target object that is not in the database +async function createPseudo(data) { + assert.strictEqual(typeof data, 'object'); + + const { id, provider, config, format } = data; // required + const encryptionPassword = data.encryptionPassword || null, + encryptedFilenames = data.encryptedFilenames || false; + + const formatError = backupFormat.validateFormat(format); + if (formatError) throw formatError; + + let encryption = null; + if (encryptionPassword) { + const encryptionPasswordError = validateEncryptionPassword(encryptionPassword); + if (encryptionPasswordError) throw encryptionPasswordError; + encryption = hush.generateEncryptionKeysSync(encryptionPassword); + encryption.encryptedFilenames = !!encryptedFilenames; + } + + debug('add: validating new storage configuration'); + const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config }); + return { id, format, provider, config: sanitizedConfig, encryption }; +} diff --git a/src/backuptask.js b/src/backuptask.js index 779d0d95b..456da8214 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -99,8 +99,8 @@ async function download(backupTarget, remotePath, dataLayout, progressCallback) await backupFormat.api(backupTarget.format).download(backupTarget, remotePath, dataLayout, progressCallback); } -async function restore(backupConfig, remotePath, progressCallback) { - assert.strictEqual(typeof backupConfig, 'object'); +async function restore(backupTarget, remotePath, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); @@ -108,7 +108,7 @@ async function restore(backupConfig, remotePath, progressCallback) { if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`); const dataLayout = new DataLayout(boxDataDir, []); - await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback); + await download(backupTarget, remotePath, dataLayout, progressCallback); debug('restore: download completed, importing database'); @@ -463,8 +463,9 @@ async function backupMailWithTag(backupTarget, tag, options, progressCallback) { return await rotateMailBackup(backupTarget, tag, options, progressCallback); } -async function downloadMail(restoreConfig, progressCallback) { - assert.strictEqual(typeof restoreConfig, 'object'); +async function downloadMail(backupTarget, remotePath, progressCallback) { + assert.strictEqual(typeof backupTarget, 'object'); + assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof progressCallback, 'function'); const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR); @@ -473,7 +474,7 @@ async function downloadMail(restoreConfig, progressCallback) { const startTime = new Date(); - await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback); + await download(backupTarget, remotePath, dataLayout, progressCallback); debug('downloadMail: time: %s', (new Date() - startTime)/1000); } diff --git a/src/paths.js b/src/paths.js index 4eee34137..8e9a8190f 100644 --- a/src/paths.js +++ b/src/paths.js @@ -28,7 +28,7 @@ exports = module.exports = { DEFAULT_BACKUP_DIR: '/var/backups', VOLUMES_MOUNT_DIR: '/mnt/volumes', - MANAGED_BACKUP_MOUNT_DIR: '/mnt/cloudronbackup', + MANAGED_BACKUP_MOUNT_DIR: '/mnt/backups', DOCKER_SOCKET_PATH: '/var/run/docker.sock', PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'), diff --git a/src/provision.js b/src/provision.js index 28b8f413d..9263b85a7 100644 --- a/src/provision.js +++ b/src/provision.js @@ -20,7 +20,6 @@ const appstore = require('./appstore.js'), domains = require('./domains.js'), eventlog = require('./eventlog.js'), fs = require('fs'), - hush = require('./hush.js'), mail = require('./mail.js'), mailServer = require('./mailserver.js'), network = require('./network.js'), @@ -172,8 +171,8 @@ async function activate(username, password, email, displayName, ip, auditSource) }; } -async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, options, auditSource) { - assert.strictEqual(typeof backupConfig, 'object'); +async function restoreTask(backupTarget, remotePath, ipv4Config, ipv6Config, options, auditSource) { + assert.strictEqual(typeof backupTarget, 'object'); assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof ipv4Config, 'object'); assert.strictEqual(typeof ipv6Config, 'object'); @@ -181,15 +180,17 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt assert.strictEqual(typeof auditSource, 'object'); try { + setProgress('restore', 'Preparing backup target'); + await backupTargets.storageApi(backupTarget).setup(backupTarget.config); + setProgress('restore', 'Downloading box backup'); - backupConfig.rootPath = backupTargets.getRootPath(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR); - await backuptask.restore(backupConfig, remotePath, (progress) => setProgress('restore', progress.message)); + await backuptask.restore(backupTarget, remotePath, (progress) => setProgress('restore', progress.message)); setProgress('restore', 'Downloading mail backup'); const mailBackups = await backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1); if (mailBackups.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'mail backup not found'); - const mailRestoreConfig = { backupConfig, remotePath: mailBackups[0].remotePath, backupFormat: mailBackups[0].format }; - await backuptask.downloadMail(mailRestoreConfig, (progress) => setProgress('restore', progress.message)); + const mailRemotePath = mailBackups[0].remotePath; + await backuptask.downloadMail(backupTarget, mailRemotePath, (progress) => setProgress('restore', progress.message)); await ensureDhparams(); await network.setIPv4Config(ipv4Config); @@ -204,8 +205,6 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt } await dashboard.setupLocation(location.subdomain, location.domain, auditSource); - delete backupConfig.rootPath; - await backupTargets.setConfig(backupConfig); await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { remotePath }); setImmediate(() => safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug })); @@ -236,24 +235,19 @@ async function restore(backupConfig, remotePath, version, ipv4Config, ipv6Config const activated = await users.isActivated(); if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.'); - let error = backupTargets.validateFormat(backupConfig.format); + const backupTarget = await backupTargets.createPseudo({ + id: `cloudron-restore`, + provider: backupConfig.provider, + config: backupConfig.config, + format: backupConfig.format, + encryptionPassword: backupConfig.encryptionPassword, + encryptedFilenames: backupConfig.encryptedFilenames + }); + + const error = await network.testIPv4Config(ipv4Config); if (error) throw error; - if ('password' in backupConfig) { - backupConfig.encryption = hush.generateEncryptionKeysSync(backupConfig.password); - delete backupConfig.password; - } else { - backupConfig.encryption = null; - } - - await backupTargets.setupManagedStorage(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR); // this validates mountOptions - error = await backupTargets.testStorage(Object.assign({ mountPath: paths.MANAGED_BACKUP_MOUNT_DIR }, backupConfig)); // this validates provider and it's api options. requires mountPath - if (error) throw error; - - error = await network.testIPv4Config(ipv4Config); - if (error) throw error; - - safe(restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background + safe(restoreTask(backupTarget, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background } catch (error) { debug('restore: error. %o', error); gStatus.restore.active = false; diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 01ed04a28..fc68ff0ad 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -254,10 +254,11 @@ async function setup(config) { debug('setup: removing old storage configuration'); if (!mounts.isManagedProvider(config.provider)) return; - await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error + const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id); + await safe(mounts.removeMount(mountPath), { debug }); // ignore error debug('setup: setting up new storage configuration'); - await setupManagedMount(config.provider, config.mountOptions, paths.MANAGED_BACKUP_MOUNT_DIR); + await setupManagedMount(config.provider, config.mountOptions, mountPath); } async function teardown(config) { @@ -265,7 +266,8 @@ async function teardown(config) { if (!mounts.isManagedProvider(config.provider)) return; - await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error + const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id); + await safe(mounts.removeMount(mountPath), { debug }); // ignore error } async function verifyConfig({ id, provider, config }) { @@ -277,7 +279,9 @@ async function verifyConfig({ id, provider, config }) { if ('chown' in config && typeof config.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean'); if ('preserveAttributes' in config && typeof config.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean'); - let rootPath, testMountObject; + const managedMountValidationPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, `${id}-validation`); + + let rootPath; if (provider === mounts.MOUNT_TYPE_FILESYSTEM) { if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string'); const error = validateDestPath(config.backupFolder); @@ -296,8 +300,8 @@ async function verifyConfig({ id, provider, config }) { const error = mounts.validateMountOptions(provider, config.mountOptions); if (error) throw error; - testMountObject = await setupManagedMount(provider, config.mountOptions, '/mnt/backup-storage-validation'); // this validates mountOptions - rootPath = path.join('/mnt/backup-storage-validation', config.prefix); + await setupManagedMount(provider, config.mountOptions, managedMountValidationPath); + rootPath = path.join(managedMountValidationPath, config.prefix); } else if (provider === mounts.MOUNT_TYPE_MOUNTPOINT) { if (!config.mountPoint || typeof config.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string'); const error = validateDestPath(config.mountPoint); @@ -324,7 +328,7 @@ async function verifyConfig({ id, provider, config }) { throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`); } - if (testMountObject) await mounts.removeMount('/mnt/backup-storage-validation'); + if (mounts.isManagedProvider(provider)) await mounts.removeMount(managedMountValidationPath); const newConfig = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']); return { provider, id, ...newConfig };