backuptarget: pseudo target for import and restore

src/apps.js

@@ -165,7 +165,6 @@ const appTaskManager = require('./apptaskmanager.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
-    hush = require('./hush.js'),
     Location = require('./location.js'),
     locks = require('./locks.js'),
     logs = require('./logs.js'),
@@ -2350,30 +2349,22 @@ async function importApp(app, data, auditSource) {
 
     const appId = app.id;
 
-    const { remotePath, format, encryptionPassword, encryptedFilenames, config } = data;
+    const error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
-
-    let error = checkAppState(app, exports.ISTATE_PENDING_IMPORT);
     if (error) throw error;
 
-    let encryption, restoreConfig;
+    let restoreConfig;
 
     if (data.remotePath) { // if not provided, we import in-place
-        error = backupTargets.validateFormat(format);
+        const backupTarget = await backupTargets.createPseudo({
-        if (error) throw error;
+            id: `appimport-${app.id}`,
+            provider: data.provider,
+            config: data.config,
+            format: data.format,
+            encryptionPassword: data.encryptionPassword,
+            encryptedFilenames: data.encryptedFilenames
+        });
 
-        if (encryptionPassword) {
+        restoreConfig = { remotePath: data.remotePath, backupTarget };
-            encryption = hush.generateEncryptionKeysSync(encryptionPassword);
-            encryption.encryptedFilenames = !!encryptedFilenames;
-        } else {
-            encryption = null;
-        }
-
-        await backupTargets.setupManagedStorage(config, `/mnt/appimport-${app.id}`); // this validates mountOptions . this is not cleaned up, it's fine
-        config.rootPath = backupTargets.getRootPath(config, `/mnt/appimport-${app.id}`);
-        error = await backupTargets.testStorage(Object.assign({ mountPath: `/mnt/appimport-${app.id}` }, config)); // this validates provider and it's api options. requires mountPath
-        if (error) throw error;
-
-        restoreConfig = { remotePath, backupFormat: format, backupConfig: config };
     } else { // inPlace
         restoreConfig = { inPlace: true };
     }
@@ -2389,7 +2380,7 @@ async function importApp(app, data, auditSource) {
     };
     const taskId = await addTask(appId, exports.ISTATE_PENDING_IMPORT, task, auditSource);
 
-    await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app: app, remotePath, fromManifest: app.manifest, toManifest: app.manifest, taskId });
+    await eventlog.add(eventlog.ACTION_APP_IMPORT, auditSource, { app, remotePath: data.remotePath, inPlace: data.inPlace, taskId });
 
     return { taskId };
 }

src/apptask.js

@@ -27,7 +27,6 @@ const apps = require('./apps.js'),
     fs = require('fs'),
     iputils = require('./iputils.js'),
     manifestFormat = require('@cloudron/manifest-format'),
-    mounts = require('./mounts.js'),
     os = require('os'),
     path = require('path'),
     paths = require('./paths.js'),
@@ -331,13 +330,11 @@ async function installCommand(app, args, progressCallback) {
         await progressCallback({ percent: 65, message: 'Downloading backup and restoring addons' });
         await services.setupAddons(app, app.manifest.addons);
         await services.clearAddons(app, app.manifest.addons);
-        const backupConfig = restoreConfig.backupConfig;
+        const backupTarget = restoreConfig.backupTarget;
-        const mountObject = await backupTargets.setupManagedStorage(backupConfig, `/mnt/appimport-${app.id}`);
+        await backupTargets.storageApi(backupTarget).setup(backupTarget.config);
-        if (mountObject) await progressCallback({ percent: 70, message: 'Setting up mount for importing' });
-        backupConfig.rootPath = backupTargets.getRootPath(backupConfig, `/mnt/appimport-${app.id}`);
         await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 75, message: progress.message }); });
         await apps.loadConfig(app);
-        if (mountObject) await mounts.removeMount(mountObject);
+        await backupTargets.storageApi(backupTarget).teardown(backupTarget.config);
         await progressCallback({ percent: 75, message: 'Restoring addons' });
         await services.restoreAddons(app, app.manifest.addons);
     } else { // clone and restore

src/backupformat.js

@@ -2,6 +2,7 @@
 
 exports = module.exports = {
     api,
+    validateFormat,
 };
 
 const assert = require('assert'),
@@ -17,3 +18,11 @@ function api(format) {
 
     throw new BoxError(BoxError.INTERNAL_ERROR, `Undefined format ${format}`);
 }
+
+function validateFormat(format) {
+    assert.strictEqual(typeof format, 'string');
+
+    if (format === 'tgz' || format == 'rsync') return null;
+
+    return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
+}

src/backuptargets.js

@@ -22,8 +22,6 @@ exports = module.exports = {
     getSnapshotInfo,
     setSnapshotInfo,
 
-    validateFormat,
-
     getRootPath,
 
     remount,
@@ -33,9 +31,12 @@ exports = module.exports = {
     storageApi,
 
     getBackupFilePath,
+
+    createPseudo,
 };
 
 const assert = require('assert'),
+    backupFormat = require('./backupformat.js'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
@@ -160,14 +161,6 @@ function removePrivateFields(target) {
     return target;
 }
 
-function validateFormat(format) {
-    assert.strictEqual(typeof format, 'string');
-
-    if (format === 'tgz' || format == 'rsync') return null;
-
-    return new BoxError(BoxError.BAD_FIELD, 'Invalid backup format');
-}
-
 function validateLabel(label) {
     assert.strictEqual(typeof label, 'string');
 
@@ -492,7 +485,7 @@ async function add(data, auditSource) {
         encryptionPassword = data.encryptionPassword || null,
         encryptedFilenames = data.encryptedFilenames || false;
 
-    const formatError = validateFormat(format);
+    const formatError = backupFormat.validateFormat(format);
     if (formatError) throw formatError;
 
     const labelError = validateLabel(label);
@@ -522,3 +515,27 @@ async function add(data, auditSource) {
 
     return id;
 }
+
+// creates a backup target object that is not in the database
+async function createPseudo(data) {
+    assert.strictEqual(typeof data, 'object');
+
+    const { id, provider, config, format } = data; // required
+    const encryptionPassword = data.encryptionPassword || null,
+        encryptedFilenames = data.encryptedFilenames || false;
+
+    const formatError = backupFormat.validateFormat(format);
+    if (formatError) throw formatError;
+
+    let encryption = null;
+    if (encryptionPassword) {
+        const encryptionPasswordError = validateEncryptionPassword(encryptionPassword);
+        if (encryptionPasswordError) throw encryptionPasswordError;
+        encryption = hush.generateEncryptionKeysSync(encryptionPassword);
+        encryption.encryptedFilenames = !!encryptedFilenames;
+    }
+
+    debug('add: validating new storage configuration');
+    const sanitizedConfig = await storageApi({ provider }).verifyConfig({id, provider, config });
+    return { id, format, provider, config: sanitizedConfig, encryption };
+}

src/backuptask.js

@@ -99,8 +99,8 @@ async function download(backupTarget, remotePath, dataLayout, progressCallback)
     await backupFormat.api(backupTarget.format).download(backupTarget, remotePath, dataLayout, progressCallback);
 }
 
-async function restore(backupConfig, remotePath, progressCallback) {
+async function restore(backupTarget, remotePath, progressCallback) {
-    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backupTarget, 'object');
     assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof progressCallback, 'function');
 
@@ -108,7 +108,7 @@ async function restore(backupConfig, remotePath, progressCallback) {
     if (!boxDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving boxdata: ${safe.error.message}`);
     const dataLayout = new DataLayout(boxDataDir, []);
 
-    await download(backupConfig, remotePath, backupConfig.format, dataLayout, progressCallback);
+    await download(backupTarget, remotePath, dataLayout, progressCallback);
 
     debug('restore: download completed, importing database');
 
@@ -463,8 +463,9 @@ async function backupMailWithTag(backupTarget, tag, options, progressCallback) {
     return await rotateMailBackup(backupTarget, tag, options, progressCallback);
 }
 
-async function downloadMail(restoreConfig, progressCallback) {
+async function downloadMail(backupTarget, remotePath, progressCallback) {
-    assert.strictEqual(typeof restoreConfig, 'object');
+    assert.strictEqual(typeof backupTarget, 'object');
+    assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof progressCallback, 'function');
 
     const mailDataDir = safe.fs.realpathSync(paths.MAIL_DATA_DIR);
@@ -473,7 +474,7 @@ async function downloadMail(restoreConfig, progressCallback) {
 
     const startTime = new Date();
 
-    await download(restoreConfig.backupConfig, restoreConfig.remotePath, restoreConfig.backupFormat, dataLayout, progressCallback);
+    await download(backupTarget, remotePath, dataLayout, progressCallback);
     debug('downloadMail: time: %s', (new Date() - startTime)/1000);
 }
 

src/paths.js

@@ -28,7 +28,7 @@ exports = module.exports = {
 
     DEFAULT_BACKUP_DIR: '/var/backups',
     VOLUMES_MOUNT_DIR: '/mnt/volumes',
-    MANAGED_BACKUP_MOUNT_DIR: '/mnt/cloudronbackup',
+    MANAGED_BACKUP_MOUNT_DIR: '/mnt/backups',
     DOCKER_SOCKET_PATH: '/var/run/docker.sock',
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),

src/provision.js

@@ -20,7 +20,6 @@ const appstore = require('./appstore.js'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
-    hush = require('./hush.js'),
     mail = require('./mail.js'),
     mailServer = require('./mailserver.js'),
     network = require('./network.js'),
@@ -172,8 +171,8 @@ async function activate(username, password, email, displayName, ip, auditSource)
     };
 }
 
-async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, options, auditSource) {
+async function restoreTask(backupTarget, remotePath, ipv4Config, ipv6Config, options, auditSource) {
-    assert.strictEqual(typeof backupConfig, 'object');
+    assert.strictEqual(typeof backupTarget, 'object');
     assert.strictEqual(typeof remotePath, 'string');
     assert.strictEqual(typeof ipv4Config, 'object');
     assert.strictEqual(typeof ipv6Config, 'object');
@@ -181,15 +180,17 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt
     assert.strictEqual(typeof auditSource, 'object');
 
     try {
+        setProgress('restore', 'Preparing backup target');
+        await backupTargets.storageApi(backupTarget).setup(backupTarget.config);
+
         setProgress('restore', 'Downloading box backup');
-        backupConfig.rootPath = backupTargets.getRootPath(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR);
+        await backuptask.restore(backupTarget, remotePath, (progress) => setProgress('restore', progress.message));
-        await backuptask.restore(backupConfig, remotePath, (progress) => setProgress('restore', progress.message));
 
         setProgress('restore', 'Downloading mail backup');
         const mailBackups = await backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_MAIL, backups.BACKUP_STATE_NORMAL, 1, 1);
         if (mailBackups.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'mail backup not found');
-        const mailRestoreConfig = { backupConfig, remotePath: mailBackups[0].remotePath, backupFormat: mailBackups[0].format };
+        const mailRemotePath = mailBackups[0].remotePath;
-        await backuptask.downloadMail(mailRestoreConfig, (progress) => setProgress('restore', progress.message));
+        await backuptask.downloadMail(backupTarget, mailRemotePath, (progress) => setProgress('restore', progress.message));
 
         await ensureDhparams();
         await network.setIPv4Config(ipv4Config);
@@ -204,8 +205,6 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, opt
         }
         await dashboard.setupLocation(location.subdomain, location.domain, auditSource);
 
-        delete backupConfig.rootPath;
-        await backupTargets.setConfig(backupConfig);
         await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { remotePath });
 
         setImmediate(() => safe(platform.onActivated({ skipDnsSetup: options.skipDnsSetup }), { debug }));
@@ -236,24 +235,19 @@ async function restore(backupConfig, remotePath, version, ipv4Config, ipv6Config
         const activated = await users.isActivated();
         if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.');
 
-        let error = backupTargets.validateFormat(backupConfig.format);
+        const backupTarget = await backupTargets.createPseudo({
+            id: `cloudron-restore`,
+            provider: backupConfig.provider,
+            config: backupConfig.config,
+            format: backupConfig.format,
+            encryptionPassword: backupConfig.encryptionPassword,
+            encryptedFilenames: backupConfig.encryptedFilenames
+        });
+
+        const error = await network.testIPv4Config(ipv4Config);
         if (error) throw error;
 
-        if ('password' in backupConfig) {
+        safe(restoreTask(backupTarget, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background
-            backupConfig.encryption = hush.generateEncryptionKeysSync(backupConfig.password);
-            delete backupConfig.password;
-        } else {
-            backupConfig.encryption = null;
-        }
-
-        await backupTargets.setupManagedStorage(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR); // this validates mountOptions
-        error = await backupTargets.testStorage(Object.assign({ mountPath: paths.MANAGED_BACKUP_MOUNT_DIR }, backupConfig)); // this validates provider and it's api options. requires mountPath
-        if (error) throw error;
-
-        error = await network.testIPv4Config(ipv4Config);
-        if (error) throw error;
-
-        safe(restoreTask(backupConfig, remotePath, ipv4Config, ipv6Config, options, auditSource), { debug }); // now that args are validated run the task in the background
     } catch (error) {
         debug('restore: error. %o', error);
         gStatus.restore.active = false;

src/storage/filesystem.js

@@ -254,10 +254,11 @@ async function setup(config) {
     debug('setup: removing old storage configuration');
     if (!mounts.isManagedProvider(config.provider)) return;
 
-    await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error
+    const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id);
+    await safe(mounts.removeMount(mountPath), { debug }); // ignore error
 
     debug('setup: setting up new storage configuration');
-    await setupManagedMount(config.provider, config.mountOptions, paths.MANAGED_BACKUP_MOUNT_DIR);
+    await setupManagedMount(config.provider, config.mountOptions, mountPath);
 }
 
 async function teardown(config) {
@@ -265,7 +266,8 @@ async function teardown(config) {
 
     if (!mounts.isManagedProvider(config.provider)) return;
 
-    await safe(mounts.removeMount(paths.MANAGED_BACKUP_MOUNT_DIR), { debug }); // ignore error
+    const mountPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, config.id);
+    await safe(mounts.removeMount(mountPath), { debug }); // ignore error
 }
 
 async function verifyConfig({ id, provider, config }) {
@@ -277,7 +279,9 @@ async function verifyConfig({ id, provider, config }) {
     if ('chown' in config && typeof config.chown !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'chown must be boolean');
     if ('preserveAttributes' in config && typeof config.preserveAttributes !== 'boolean') throw new BoxError(BoxError.BAD_FIELD, 'preserveAttributes must be boolean');
 
-    let rootPath, testMountObject;
+    const managedMountValidationPath = path.join(paths.MANAGED_BACKUP_MOUNT_DIR, `${id}-validation`);
+
+    let rootPath;
     if (provider === mounts.MOUNT_TYPE_FILESYSTEM) {
         if (!config.backupFolder || typeof config.backupFolder !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'backupFolder must be non-empty string');
         const error = validateDestPath(config.backupFolder);
@@ -296,8 +300,8 @@ async function verifyConfig({ id, provider, config }) {
             const error = mounts.validateMountOptions(provider, config.mountOptions);
             if (error) throw error;
 
-            testMountObject = await setupManagedMount(provider, config.mountOptions, '/mnt/backup-storage-validation'); // this validates mountOptions
+            await setupManagedMount(provider, config.mountOptions, managedMountValidationPath);
-            rootPath = path.join('/mnt/backup-storage-validation', config.prefix);
+            rootPath = path.join(managedMountValidationPath, config.prefix);
         } else if (provider === mounts.MOUNT_TYPE_MOUNTPOINT) {
             if (!config.mountPoint || typeof config.mountPoint !== 'string') throw new BoxError(BoxError.BAD_FIELD, 'mountPoint must be non-empty string');
             const error = validateDestPath(config.mountPoint);
@@ -324,7 +328,7 @@ async function verifyConfig({ id, provider, config }) {
         throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`);
     }
 
-    if (testMountObject) await mounts.removeMount('/mnt/backup-storage-validation');
+    if (mounts.isManagedProvider(provider)) await mounts.removeMount(managedMountValidationPath);
 
     const newConfig = _.pick(config, ['noHardlinks', 'chown', 'preserveAttributes', 'backupFolder', 'prefix', 'mountOptions', 'mountPoint']);
     return { provider, id, ...newConfig };