diff --git a/CHANGES b/CHANGES index dbb56c890..4ce42c53e 100644 --- a/CHANGES +++ b/CHANGES @@ -2670,4 +2670,5 @@ * graphs: show old backup size if > 1GB * docker: fix image pruning * Major overhaul of the REST API +* Fix import via SSHFS and CIFS diff --git a/src/apps.js b/src/apps.js index e8de676d5..6e0f10cd0 100644 --- a/src/apps.js +++ b/src/apps.js @@ -171,7 +171,6 @@ const appstore = require('./appstore.js'), logs = require('./logs.js'), mail = require('./mail.js'), manifestFormat = require('cloudron-manifestformat'), - mounts = require('./mounts.js'), notifications = require('./notifications.js'), once = require('./once.js'), os = require('os'), @@ -2197,23 +2196,11 @@ async function importApp(app, data, auditSource) { let restoreConfig; if (data.remotePath) { // if not provided, we import in-place - error = backups.validateBackupFormat(backupFormat); + error = backups.validateFormat(backupFormat); if (error) throw error; - // TODO: make this smarter to do a read-only test and check if the file exists in the storage backend - if (mounts.isManagedProvider(backupConfig.provider)) { - error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions); - if (error) throw error; - - const mountObject = { // keep this in sync with the import code in apptask - name: `appimport-${app.id}`, - hostPath: `/mnt/appimport-${app.id}`, - mountType: backupConfig.provider, - mountOptions: backupConfig.mountOptions - }; - await mounts.tryAddMount(mountObject, { timeout: 10 }); - - } + await backups.setupStorage(backupConfig, `/mnt/appimport-${app.id}`); + backupConfig.rootPath = backups.getRootPath(backupConfig, `/mnt/appimport-${app.id}`); error = await backups.testStorage(backupConfig); if (error) throw error; diff --git a/src/apptask.js b/src/apptask.js index b75a6b039..fd48d544f 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -15,6 +15,7 @@ const apps = require('./apps.js'), appstore = require('./appstore.js'), assert = require('assert'), AuditSource = require('./auditsource.js'), + backups = require('./backups.js'), backuptask = require('./backuptask.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), @@ -319,17 +320,9 @@ async function install(app, args, progressCallback) { await services.setupAddons(app, app.manifest.addons); await services.clearAddons(app, app.manifest.addons); const backupConfig = restoreConfig.backupConfig; - let mountObject = null; - if (mounts.isManagedProvider(backupConfig.provider)) { - await progressCallback({ percent: 70, message: 'Setting up mount for importing' }); - mountObject = { // keep this in sync with importApp in apps.js - name: `appimport-${app.id}`, - hostPath: `/mnt/appimport-${app.id}`, - mountType: backupConfig.provider, - mountOptions: backupConfig.mountOptions - }; - await mounts.tryAddMount(mountObject, { timeout: 10 }); - } + const mountObject = await backups.setupStorage(backupConfig, `/mnt/appimport-${app.id}`); + if (mountObject) await progressCallback({ percent: 70, message: 'Setting up mount for importing' }); + backupConfig.rootPath = backups.getRootPath(backupConfig, `/mnt/appimport-${app.id}`); await backuptask.downloadApp(app, restoreConfig, (progress) => { progressCallback({ percent: 75, message: progress.message }); }); await apps.restoreConfig(app); if (mountObject) await mounts.removeMount(mountObject); diff --git a/src/backupformat/rsync.js b/src/backupformat/rsync.js index 939293dd2..9dc317588 100644 --- a/src/backupformat/rsync.js +++ b/src/backupformat/rsync.js @@ -29,8 +29,7 @@ function getBackupFilePath(backupConfig, remotePath) { assert.strictEqual(typeof backupConfig, 'object'); assert.strictEqual(typeof remotePath, 'string'); - const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig); - return path.join(rootPath, remotePath); + return path.join(backupConfig.rootPath, remotePath); } function sync(backupConfig, remotePath, dataLayout, progressCallback, callback) { diff --git a/src/backupformat/tgz.js b/src/backupformat/tgz.js index ab6785e8e..9dff4fc01 100644 --- a/src/backupformat/tgz.js +++ b/src/backupformat/tgz.js @@ -24,8 +24,7 @@ function getBackupFilePath(backupConfig, remotePath) { assert.strictEqual(typeof backupConfig, 'object'); assert.strictEqual(typeof remotePath, 'string'); - const rootPath = storage.api(backupConfig.provider).getBackupRootPath(backupConfig); - + const rootPath = backupConfig.rootPath; const fileType = backupConfig.encryption ? '.tar.gz.enc' : '.tar.gz'; return path.join(rootPath, remotePath + fileType); } diff --git a/src/backups.js b/src/backups.js index 415e8f777..5981c4030 100644 --- a/src/backups.js +++ b/src/backups.js @@ -15,7 +15,6 @@ exports = module.exports = { startCleanupTask, cleanupCacheFilesSync, - injectPrivateFields, removePrivateFields, generateEncryptionKeysSync, @@ -26,7 +25,7 @@ exports = module.exports = { validatePolicy, validateEncryptionPassword, testStorage, - validateBackupFormat, + validateFormat, getPolicy, setPolicy, @@ -36,6 +35,8 @@ exports = module.exports = { setStorage, setLimits, + setupStorage, + remount, getMountStatus, @@ -85,24 +86,13 @@ function postProcess(result) { return result; } -function injectPrivateFields(newConfig, currentConfig) { - if ('password' in newConfig) { - if (newConfig.password === constants.SECRET_PLACEHOLDER) { - delete newConfig.password; - } - newConfig.encryption = currentConfig.encryption || null; - } else { - newConfig.encryption = null; - } - if (newConfig.provider === currentConfig.provider) storage.api(newConfig.provider).injectPrivateFields(newConfig, currentConfig); -} - function removePrivateFields(backupConfig) { assert.strictEqual(typeof backupConfig, 'object'); if (backupConfig.encryption) { delete backupConfig.encryption; backupConfig.password = constants.SECRET_PLACEHOLDER; } + delete backupConfig.rootPath; return storage.api(backupConfig.provider).removePrivateFields(backupConfig); } @@ -414,10 +404,26 @@ async function setPolicy(policy) { await cron.handleBackupPolicyChanged(policy); } +function getRootPath(storageConfig, mountPath) { + assert.strictEqual(typeof storageConfig, 'object'); + assert.strictEqual(typeof mountPath, 'string'); + + if (mounts.isManagedProvider(storageConfig.provider)) { + return path.join(mountPath, storageConfig.prefix); + } else if (storageConfig.provider === 'mountpoint') { + return path.join(storageConfig.mountPoint, storageConfig.prefix); + } else if (storageConfig.provider === 'filesystem') { + return storageConfig.backupFolder; + } else { + return storageConfig.prefix; + } +} + async function getConfig() { const result = await settings.getJson(settings.BACKUP_STORAGE_KEY) || { provider: 'filesystem', backupFolder: paths.DEFAULT_BACKUP_DIR, format: 'tgz', encryption: null }; const limits = await settings.getJson(settings.BACKUP_LIMITS_KEY); if (limits) result.limits = limits; + result.rootPath = getRootPath(result, paths.MANAGED_BACKUP_MOUNT_DIR); // note: rootPath will be dynamic for managed mount providers during app import return result; } @@ -434,7 +440,7 @@ async function setLimits(limits) { await settings.setJson(settings.BACKUP_LIMITS_KEY, limits); } -function validateBackupFormat(format) { +function validateFormat(format) { assert.strictEqual(typeof format, 'string'); if (format === 'tgz' || format == 'rsync') return null; @@ -447,36 +453,34 @@ async function setStorage(storageConfig) { const oldConfig = await getConfig(); - injectPrivateFields(storageConfig, oldConfig); + if (storageConfig.provider === oldConfig.provider) storage.api(storageConfig.provider).injectPrivateFields(storageConfig, oldConfig); - if (mounts.isManagedProvider(storageConfig.provider)) { - let error = mounts.validateMountOptions(storageConfig.provider, storageConfig.mountOptions); - if (error) throw error; - - [error] = await safe(mounts.tryAddMount(mountObjectFromBackupConfig(storageConfig), { timeout: 10 })); // 10 seconds - - if (error) { - if (mounts.isManagedProvider(oldConfig.provider)) { // put back the old mount configuration - debug('setBackupConfig: rolling back to previous mount configuration'); - - await safe(mounts.tryAddMount(mountObjectFromBackupConfig(oldConfig), { timeout: 10 })); - } - - throw error; - } - } - - let error = validateBackupFormat(storageConfig.format); + let error = validateFormat(storageConfig.format); if (error) throw error; + debug('setStorage: validating new storage configuration'); + await setupStorage(storageConfig, '/mnt/backup-storage-validation'); + storageConfig.rootPath = getRootPath(storageConfig, '/mnt/backup-storage-validation'); error = await testStorage(storageConfig); + delete storageConfig.rootPath; if (error) throw error; - if ('password' in storageConfig) { // user set password - const error = await validateEncryptionPassword(storageConfig.password); - if (error) throw error; + debug('setStorage: removing old storage configuration'); + if (mounts.isManagedProvider(oldConfig.provider)) await safe(mounts.removeMount(mountObjectFromBackupConfig(oldConfig))); - storageConfig.encryption = generateEncryptionKeysSync(storageConfig.password); + debug('setStorage: setting up new storage configuration'); + await setupStorage(storageConfig, paths.MANAGED_BACKUP_MOUNT_DIR); + + storageConfig.encryption = null; + if ('password' in storageConfig) { // user set password + if (storageConfig.password === constants.SECRET_PLACEHOLDER) { + storageConfig.encryption = oldConfig.encryption || null; + } else { + const error = await validateEncryptionPassword(storageConfig.password); + if (error) throw error; + + storageConfig.encryption = generateEncryptionKeysSync(storageConfig.password); + } delete storageConfig.password; } @@ -484,9 +488,25 @@ async function setStorage(storageConfig) { cleanupCacheFilesSync(); await settings.setJson(settings.BACKUP_STORAGE_KEY, storageConfig); - - if (mounts.isManagedProvider(oldConfig.provider) && !mounts.isManagedProvider(storageConfig.provider)) { - debug('setBackupConfig: removing old backup mount point'); - await safe(mounts.removeMount(mountObjectFromBackupConfig(oldConfig))); - } +} + +async function setupStorage(storageConfig, hostPath) { + assert.strictEqual(typeof storageConfig, 'object'); + assert.strictEqual(typeof hostPath, 'string'); + + if (!mounts.isManagedProvider(storageConfig.provider)) return null; + + const error = mounts.validateMountOptions(storageConfig.provider, storageConfig.mountOptions); + if (error) throw error; + + const newMount = { + name: path.basename(hostPath), + hostPath: hostPath, + mountType: storageConfig.provider, + mountOptions: storageConfig.mountOptions + }; + + await mounts.tryAddMount(newMount, { timeout: 10 }); // 10 seconds + + return newMount; } diff --git a/src/provision.js b/src/provision.js index 9c8888228..016d6dcf6 100644 --- a/src/provision.js +++ b/src/provision.js @@ -20,7 +20,6 @@ const assert = require('assert'), fs = require('fs'), mail = require('./mail.js'), mailServer = require('./mailserver.js'), - mounts = require('./mounts.js'), network = require('./network.js'), platform = require('./platform.js'), reverseProxy = require('./reverseproxy.js'), @@ -190,6 +189,7 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, options, auditS } await dashboard.setLocation(location.domain, auditSource); + delete backupConfig.rootPath; await backups.setConfig(backupConfig); await eventlog.add(eventlog.ACTION_RESTORE, auditSource, { remotePath }); @@ -202,7 +202,7 @@ async function restoreTask(backupConfig, remotePath, ipv4Config, options, auditS } async function restore(backupConfig, remotePath, version, ipv4Config, options, auditSource) { - assert.strictEqual(typeof backupConfig, 'object'); + assert.strictEqual(typeof backupConfig, 'object'); // format, storage, password assert.strictEqual(typeof remotePath, 'string'); assert.strictEqual(typeof version, 'string'); assert.strictEqual(typeof ipv4Config, 'object'); @@ -220,21 +220,12 @@ async function restore(backupConfig, remotePath, version, ipv4Config, options, a const activated = await users.isActivated(); if (activated) throw new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.'); - if (mounts.isManagedProvider(backupConfig.provider)) { - const error = mounts.validateMountOptions(backupConfig.provider, backupConfig.mountOptions); - if (error) throw error; + let error = backups.validateFormat(backupConfig.format); + if (error) throw error; - const newMount = { - name: 'backup', - hostPath: paths.MANAGED_BACKUP_MOUNT_DIR, - mountType: backupConfig.provider, - mountOptions: backupConfig.mountOptions - }; - - await mounts.tryAddMount(newMount, { timeout: 10 }); // 10 seconds - } - - let error = await backups.testStorage(backupConfig); + await backups.setupStorage(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR); + backupConfig.rootPath = backups.getRootPath(backupConfig, paths.MANAGED_BACKUP_MOUNT_DIR); + error = await backups.testStorage(backupConfig); if (error) throw error; if ('password' in backupConfig) { diff --git a/src/routes/test/backups-test.js b/src/routes/test/backups-test.js index 4ff280a3d..6e888dcd7 100644 --- a/src/routes/test/backups-test.js +++ b/src/routes/test/backups-test.js @@ -228,7 +228,7 @@ describe('Backups API', function () { tmp.backupFolder = BACKUP_FOLDER; tmp.limits = { copyConcurrency: 34 }; - const response = await superagent.post(`${serverUrl}/api/v1/backups/config`) + const response = await superagent.post(`${serverUrl}/api/v1/backups/config/storage`) .query({ access_token: owner.token }) .send(tmp); @@ -247,7 +247,7 @@ describe('Backups API', function () { describe('create', function () { before(async function () { - await backups.setConfig({ + await backups.setStorage({ provider: 'filesystem', backupFolder: '/tmp/backups', format: 'tgz', diff --git a/src/storage/filesystem.js b/src/storage/filesystem.js index 39f678242..7240db556 100644 --- a/src/storage/filesystem.js +++ b/src/storage/filesystem.js @@ -1,7 +1,6 @@ 'use strict'; exports = module.exports = { - getBackupRootPath, getProviderStatus, getAvailableSize, @@ -43,25 +42,6 @@ const assert = require('assert'), safe = require('safetydance'), shell = require('../shell.js'); -// storage api -function getBackupRootPath(apiConfig) { - assert.strictEqual(typeof apiConfig, 'object'); - - switch (apiConfig.provider) { - case PROVIDER_SSHFS: - case PROVIDER_NFS: - case PROVIDER_CIFS: - case PROVIDER_EXT4: - case PROVIDER_XFS: - case PROVIDER_DISK: - return path.join(paths.MANAGED_BACKUP_MOUNT_DIR, apiConfig.prefix); - case PROVIDER_MOUNTPOINT: - return path.join(apiConfig.mountPoint, apiConfig.prefix); - case PROVIDER_FILESYSTEM: - return apiConfig.backupFolder; - } -} - async function getProviderStatus(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); @@ -76,7 +56,7 @@ async function getProviderStatus(apiConfig) { async function getAvailableSize(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); - const [error, dfResult] = await safe(df.file(getBackupRootPath(apiConfig))); + const [error, dfResult] = await safe(df.file(apiConfig.rootPath)); if (error) throw new BoxError(BoxError.FS_ERROR, `Error when checking for disk space: ${error.message}`); return dfResult.available; @@ -280,20 +260,20 @@ async function testConfig(apiConfig) { if (!safe.child_process.execSync(`mountpoint -q -- ${apiConfig.mountPoint}`)) throw new BoxError(BoxError.BAD_FIELD, `${apiConfig.mountPoint} is not mounted`); } - const basePath = getBackupRootPath(apiConfig); + const rootPath = apiConfig.rootPath; const field = apiConfig.provider === PROVIDER_FILESYSTEM ? 'backupFolder' : 'mountPoint'; - if (!safe.fs.mkdirSync(path.join(basePath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') { - if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create the directory and run "chown yellowtent:yellowtent ${basePath}" on the server`, { field }); + if (!safe.fs.mkdirSync(path.join(rootPath, 'snapshot'), { recursive: true }) && safe.error.code !== 'EEXIST') { + if (safe.error && safe.error.code === 'EACCES') throw new BoxError(BoxError.BAD_FIELD, `Access denied. Create the directory and run "chown yellowtent:yellowtent ${rootPath}" on the server`, { field }); throw new BoxError(BoxError.BAD_FIELD, safe.error.message, { field }); } - if (!safe.fs.writeFileSync(path.join(basePath, 'cloudron-testfile'), 'testcontent')) { - throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${basePath}: ${safe.error.message}. Check dir/mount permissions`, { field }); + if (!safe.fs.writeFileSync(path.join(rootPath, 'cloudron-testfile'), 'testcontent')) { + throw new BoxError(BoxError.BAD_FIELD, `Unable to create test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`, { field }); } - if (!safe.fs.unlinkSync(path.join(basePath, 'cloudron-testfile'))) { - throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${basePath}: ${safe.error.message}. Check dir/mount permissions`, { field }); + if (!safe.fs.unlinkSync(path.join(rootPath, 'cloudron-testfile'))) { + throw new BoxError(BoxError.BAD_FIELD, `Unable to remove test file as 'yellowtent' user in ${rootPath}: ${safe.error.message}. Check dir/mount permissions`, { field }); } } diff --git a/src/storage/gcs.js b/src/storage/gcs.js index 17893a9da..655499541 100644 --- a/src/storage/gcs.js +++ b/src/storage/gcs.js @@ -1,7 +1,6 @@ 'use strict'; exports = module.exports = { - getBackupRootPath, getProviderStatus, getAvailableSize, @@ -62,13 +61,6 @@ function getBucket(apiConfig) { return new GCS(gcsConfig).bucket(apiConfig.bucket); } -// storage api -function getBackupRootPath(apiConfig) { - assert.strictEqual(typeof apiConfig, 'object'); - - return apiConfig.prefix; -} - async function getProviderStatus(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/storage/interface.js b/src/storage/interface.js index 6a01a64ee..c69fb95e6 100644 --- a/src/storage/interface.js +++ b/src/storage/interface.js @@ -11,7 +11,6 @@ // for the other API calls we leave it to the backend to retry. this allows // them to tune the concurrency based on failures/rate limits accordingly exports = module.exports = { - getBackupRootPath, getProviderStatus, getAvailableSize, @@ -45,13 +44,6 @@ function injectPrivateFields(newConfig, currentConfig) { // in-place injection of tokens and api keys which came in with constants.SECRET_PLACEHOLDER } -function getBackupRootPath(apiConfig) { - assert.strictEqual(typeof apiConfig, 'object'); - - // Result: path at the backup storage - return '/'; -} - async function getProviderStatus(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/storage/noop.js b/src/storage/noop.js index 8afbb7b1f..fa7b92a69 100644 --- a/src/storage/noop.js +++ b/src/storage/noop.js @@ -1,7 +1,6 @@ 'use strict'; exports = module.exports = { - getBackupRootPath, getProviderStatus, getAvailableSize, @@ -24,11 +23,6 @@ const assert = require('assert'), BoxError = require('../boxerror.js'), debug = require('debug')('box:storage/noop'); -function getBackupRootPath(apiConfig) { - assert.strictEqual(typeof apiConfig, 'object'); - return ''; -} - async function getProviderStatus(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/storage/s3.js b/src/storage/s3.js index c04470901..2a0a3ffb1 100644 --- a/src/storage/s3.js +++ b/src/storage/s3.js @@ -1,7 +1,6 @@ 'use strict'; exports = module.exports = { - getBackupRootPath, getProviderStatus, getAvailableSize, @@ -92,13 +91,6 @@ function getS3Config(apiConfig) { return credentials; } -// storage api -function getBackupRootPath(apiConfig) { - assert.strictEqual(typeof apiConfig, 'object'); - - return apiConfig.prefix; -} - async function getProviderStatus(apiConfig) { assert.strictEqual(typeof apiConfig, 'object'); diff --git a/src/test/storage-test.js b/src/test/storage-test.js index 0ee4d2aaa..52cbe82d2 100644 --- a/src/test/storage-test.js +++ b/src/test/storage-test.js @@ -53,14 +53,14 @@ describe('Storage', function () { done(); }); - it('fails to set backup config for bad folder', async function () { + it('fails to set backup storage for bad folder', async function () { const tmp = Object.assign({}, gBackupConfig, { backupFolder: '/root/oof' }); - const [error] = await safe(backups.setConfig(tmp)); + const [error] = await safe(backups.setStorage(tmp)); expect(error.reason).to.equal(BoxError.BAD_FIELD); }); - it('succeeds to set backup config', async function () { - await backups.setConfig(gBackupConfig); + it('succeeds to set backup storage', async function () { + await backups.setStorage(gBackupConfig); expect(fs.existsSync(path.join(gBackupConfig.backupFolder, 'snapshot'))).to.be(true); // auto-created });