diff --git a/CHANGES b/CHANGES index ffe65dc91..ee20b160f 100644 --- a/CHANGES +++ b/CHANGES @@ -2495,4 +2495,5 @@ * graphite: fix issue where disk names with '.' do not render * dark mode fixes * sendmail: mail from display name +* Use volumes for app data instead of raw path diff --git a/migrations/20220602050517-apps-add-storageVolumeId.js b/migrations/20220602050517-apps-add-storageVolumeId.js new file mode 100644 index 000000000..77149aab9 --- /dev/null +++ b/migrations/20220602050517-apps-add-storageVolumeId.js @@ -0,0 +1,50 @@ +'use strict'; + +const path = require('path'), + safe = require('safetydance'), + uuid = require('uuid'); + +function getMountPoint(dataDir) { + const output = safe.child_process.execSync(`df --output=target "${dataDir}" | tail -1`, { encoding: 'utf8' }); + if (!output) return dataDir; + const mountPoint = output.trim(); + if (mountPoint === '/') return dataDir; + return mountPoint; +} + +exports.up = async function(db) { + await db.runSql('ALTER TABLE apps ADD storageVolumeId VARCHAR(128), ADD FOREIGN KEY(storageVolumeId) REFERENCES volumes(id)'); + await db.runSql('ALTER TABLE apps ADD storageVolumePrefix VARCHAR(128)'); + await db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_storageVolume UNIQUE (storageVolumeId, storageVolumePrefix)'); + + const apps = await db.runSql('SELECT * FROM apps WHERE dataDir IS NOT NULL'); + const allVolumes = await db.runSql('SELECT * FROM volumes'); + + for (const app of apps) { + console.log(`data-dir (${app.id}): migrating data dir ${app.dataDir}`); + + const mountPoint = getMountPoint(app.dataDir); + const prefix = path.relative(mountPoint, app.dataDir); + + console.log(`data-dir (${app.id}): migrating to mountpoint ${mountPoint} and prefix ${prefix}`); + + const volume = allVolumes.find(v => v.hostPath === mountPoint); + if (volume) { + console.log(`data-dir (${app.id}): using existing volume ${volume.id}`); + await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ volume.id, prefix, app.id ]); + continue; + } + + const id = uuid.v4().replace(/-/g, ''); // to make systemd mount file names more readable + const name = `app-${app.id}`; + + console.log(`data-dir (${app.id}): creating new volume ${id}`); + await db.runSql('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, mountPoint, 'mountpoint', JSON.stringify({}) ]); + await db.runSql('UPDATE apps SET storageVolumeId=?, storageVolumePrefix=? WHERE id=?', [ id, prefix, app.id ]); + } + + await db.runSql('ALTER TABLE apps DROP COLUMN dataDir'); +}; + +exports.down = async function(/*db*/) { +}; diff --git a/migrations/schema.sql b/migrations/schema.sql index af70b84b8..f418ce488 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -93,7 +93,8 @@ CREATE TABLE IF NOT EXISTS apps( inboxDomain VARCHAR(128), // mailbox domain of this app label VARCHAR(128), // display name tagsJson VARCHAR(2048), // array of tags - dataDir VARCHAR(256) UNIQUE, + storageVolumeId VARCHAR(128), + storageVolumePrefix VARCHAR(128), taskId INTEGER, // current task errorJson TEXT, servicesConfigJson TEXT, // app services configuration @@ -104,6 +105,8 @@ CREATE TABLE IF NOT EXISTS apps( FOREIGN KEY(mailboxDomain) REFERENCES domains(domain), FOREIGN KEY(taskId) REFERENCES tasks(id), + FOREIGN KEY(storageVolumeId) REFERENCES volumes(id), + UNIQUE (storageVolumeId, storageVolumePrefix), PRIMARY KEY(id)); CREATE TABLE IF NOT EXISTS appPortBindings( diff --git a/src/apps.js b/src/apps.js index bce2845b6..6a623b849 100644 --- a/src/apps.js +++ b/src/apps.js @@ -42,7 +42,7 @@ exports = module.exports = { setMailbox, setInbox, setLocation, - setDataDir, + setStorage, repair, restore, @@ -81,7 +81,7 @@ exports = module.exports = { schedulePendingTasks, restartAppsUsingAddons, - getDataDir, + getStorageDir, getIcon, getMemoryLimit, getLimits, @@ -178,6 +178,7 @@ const appstore = require('./appstore.js'), util = require('util'), uuid = require('uuid'), validator = require('validator'), + volumes = require('./volumes.js'), _ = require('underscore'); const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState', @@ -186,7 +187,7 @@ const APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationS 'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', 'apps.crontab', 'apps.creationTime', 'apps.updateTime', 'apps.enableAutomaticUpdate', 'apps.enableMailbox', 'apps.mailboxDisplayName', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableInbox', 'apps.inboxName', 'apps.inboxDomain', - 'apps.dataDir', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(','); + 'apps.storageVolumeId', 'apps.storageVolumePrefix', 'apps.ts', 'apps.healthTime', '(apps.icon IS NOT NULL) AS hasIcon', '(apps.appStoreIcon IS NOT NULL) AS hasAppStoreIcon' ].join(','); // const PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(','); @@ -477,28 +478,15 @@ function validateEnv(env) { return null; } -function validateDataDir(dataDir) { - if (dataDir === null) return null; +function validateStorage(volume, prefix) { + assert.strictEqual(typeof volume, 'object'); + assert.strictEqual(typeof prefix, 'string'); - if (!path.isAbsolute(dataDir)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not an absolute path`); - if (dataDir.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `${dataDir} contains trailing slash`); - if (path.normalize(dataDir) !== dataDir) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a normalized path`); + // TODO: check the volume type - // nfs shares will have the directory mounted already - let stat = safe.fs.lstatSync(dataDir); - if (stat) { - if (!stat.isDirectory()) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not a directory`); - let entries = safe.fs.readdirSync(dataDir); - if (!entries) return new BoxError(BoxError.BAD_FIELD, `${dataDir} could not be listed`); - if (entries.length !== 0) return new BoxError(BoxError.BAD_FIELD, `${dataDir} is not empty. If this is the root of a mounted volume, provide a subdirectory.`); - } - - // backup logic relies on paths not overlapping (because it recurses) - if (dataDir.startsWith(paths.APPS_DATA_DIR)) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be inside apps data`); - - // if we made it this far, it cannot start with any of these realistically - const fhs = [ '/bin', '/boot', '/etc', '/lib', '/lib32', '/lib64', '/proc', '/run', '/sbin', '/tmp', '/usr' ]; - if (fhs.some((p) => dataDir.startsWith(p))) return new BoxError(BoxError.BAD_FIELD, `${dataDir} cannot be placed inside this location`); + if (path.isAbsolute(prefix)) return new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" must be a relative path`); + if (prefix.endsWith('/')) return new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" contains trailing slash`); + if (path.normalize(prefix) !== prefix) return new BoxError(BoxError.BAD_FIELD, `prefix "${prefix}" is not a normalized path`); return null; } @@ -530,17 +518,20 @@ function getDuplicateErrorDetails(errorMessage, locations, domainObjectMap, port if (portBindings[portName] === parseInt(match[1])) return new BoxError(BoxError.ALREADY_EXISTS, `Port ${match[1]} is in use`); } - if (match[2] === 'dataDir') { - return new BoxError(BoxError.BAD_FIELD, `Data directory ${match[1]} is in use`); + if (match[2] === 'apps_storageVolume') { + return new BoxError(BoxError.BAD_FIELD, `Storage directory ${match[1]} is in use`); } return new BoxError(BoxError.ALREADY_EXISTS, `${match[2]} '${match[1]}' is in use`); } -async function getDataDir(app, dataDir) { - assert(dataDir === null || typeof dataDir === 'string'); +async function getStorageDir(app) { + assert.strictEqual(typeof app, 'object'); - return dataDir || path.join(paths.APPS_DATA_DIR, app.id, 'data'); + if (!app.storageVolumeId) return path.join(paths.APPS_DATA_DIR, app.id, 'data'); + const volume = await volumes.get(app.storageVolumeId); + if (!volume) throw new BoxError(BoxError.NOT_FOUND, 'Volume not found'); // not possible + return path.join(volume.hostPath, app.storageVolumePrefix); } function removeInternalFields(app) { @@ -549,7 +540,7 @@ function removeInternalFields(app) { 'subdomain', 'domain', 'fqdn', 'crontab', 'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'operators', 'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags', - 'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts', + 'label', 'secondaryDomains', 'redirectDomains', 'aliasDomains', 'env', 'enableAutomaticUpdate', 'storageVolumeId', 'storageVolumePrefix', 'mounts', 'enableMailbox', 'mailboxDisplayName', 'mailboxName', 'mailboxDomain', 'enableInbox', 'inboxName', 'inboxDomain'); } @@ -1794,25 +1785,31 @@ async function setLocation(app, data, auditSource) { return { taskId }; } -async function setDataDir(app, dataDir, auditSource) { +async function setStorage(app, volumeId, volumePrefix, auditSource) { assert.strictEqual(typeof app, 'object'); - assert(dataDir === null || typeof dataDir === 'string'); + assert(volumeId === null || typeof volumeId === 'string'); + assert(volumePrefix === null || typeof volumePrefix === 'string'); assert.strictEqual(typeof auditSource, 'object'); const appId = app.id; let error = checkAppState(app, exports.ISTATE_PENDING_DATA_DIR_MIGRATION); if (error) throw error; - error = validateDataDir(dataDir); - if (error) throw error; + if (volumeId) { + const volume = await volumes.get(volumeId); + if (volume === null) return new BoxError(BoxError.BAD_FIELD, 'Storage volume not found'); + + error = validateStorage(volume, volumePrefix); + if (error) throw error; + } const task = { - args: { newDataDir: dataDir }, + args: { newStorageVolumeId: volumeId, newStorageVolumePrefix: volumePrefix }, values: {} }; const taskId = await addTask(appId, exports.ISTATE_PENDING_DATA_DIR_MIGRATION, task, auditSource); - await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, dataDir, taskId }); + await eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, volumeId, volumePrefix, taskId }); return { taskId }; } diff --git a/src/apptask.js b/src/apptask.js index b9a1fa177..642a5812b 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -160,7 +160,7 @@ async function deleteAppDir(app, options) { async function addCollectdProfile(app) { assert.strictEqual(typeof app, 'object'); - const appDataDir = await apps.getDataDir(app, app.dataDir); + const appDataDir = await apps.getStorageDir(app); const collectdConf = ejs.render(COLLECTD_CONFIG_EJS, { appId: app.id, containerId: app.containerId, appDataDir }); await collectd.addProfile(app.id, collectdConf); } @@ -269,12 +269,13 @@ async function waitForDnsPropagation(app) { } } -async function moveDataDir(app, targetDir) { +async function moveDataDir(app, targetVolumeId, targetVolumePrefix) { assert.strictEqual(typeof app, 'object'); - assert(targetDir === null || typeof targetDir === 'string'); + assert(targetVolumeId === null || typeof targetVolumeId === 'string'); + assert(targetVolumePrefix === null || typeof targetVolumePrefix === 'string'); - const resolvedSourceDir = await apps.getDataDir(app, app.dataDir); - const resolvedTargetDir = await apps.getDataDir(app, targetDir); + const resolvedSourceDir = await apps.getStorageDir(app); + const resolvedTargetDir = await apps.getStorageDir(_.extend({}, app, { storageVolumeId: targetVolumeId, storageVolumePrefix: targetVolumePrefix })); debug(`moveDataDir: migrating data from ${resolvedSourceDir} to ${resolvedTargetDir}`); @@ -521,8 +522,9 @@ async function migrateDataDir(app, args, progressCallback) { assert.strictEqual(typeof args, 'object'); assert.strictEqual(typeof progressCallback, 'function'); - const newDataDir = args.newDataDir; - assert(newDataDir === null || typeof newDataDir === 'string'); + const { newStorageVolumeId, newStorageVolumePrefix } = args; + assert(newStorageVolumeId === null || typeof newStorageVolumeId === 'string'); + assert(newStorageVolumePrefix === null || typeof newStorageVolumePrefix === 'string'); await progressCallback({ percent: 10, message: 'Cleaning up old install' }); await deleteContainers(app, { managedOnly: true }); @@ -530,12 +532,12 @@ async function migrateDataDir(app, args, progressCallback) { await progressCallback({ percent: 45, message: 'Ensuring app data directory' }); await createAppDir(app); - // re-setup addons since this creates the localStorage volume + // re-setup addons since this creates the localStorage destination await progressCallback({ percent: 50, message: 'Setting up addons' }); - await services.setupAddons(_.extend({}, app, { dataDir: newDataDir }), app.manifest.addons); + await services.setupAddons(_.extend({}, app, { storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }), app.manifest.addons); await progressCallback({ percent: 60, message: 'Moving data dir' }); - await moveDataDir(app, newDataDir); + await moveDataDir(app, newStorageVolumeId, newStorageVolumePrefix); await progressCallback({ percent: 90, message: 'Creating container' }); await createContainer(app); @@ -543,7 +545,7 @@ async function migrateDataDir(app, args, progressCallback) { await startApp(app); await progressCallback({ percent: 100, message: 'Done' }); - await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir }); + await updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, storageVolumeId: newStorageVolumeId, storageVolumePrefix: newStorageVolumePrefix }); } // configure is called for an infra update and repair to re-create container, reverseproxy config. it's all "local" diff --git a/src/backuptask.js b/src/backuptask.js index a4ee02c75..05c67e704 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -102,7 +102,7 @@ async function downloadApp(app, restoreConfig, progressCallback) { const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, safe.error.message); - const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []); + const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []); const startTime = new Date(); const backupConfig = restoreConfig.backupConfig || await settings.getBackupConfig(); @@ -318,7 +318,7 @@ async function uploadAppSnapshot(backupConfig, app, progressCallback) { const appDataDir = safe.fs.realpathSync(path.join(paths.APPS_DATA_DIR, app.id)); if (!appDataDir) throw new BoxError(BoxError.FS_ERROR, `Error resolving appsdata: ${safe.error.message}`); - const dataLayout = new DataLayout(appDataDir, app.dataDir ? [{ localDir: app.dataDir, remoteDir: 'data' }] : []); + const dataLayout = new DataLayout(appDataDir, app.storageVolumeId ? [{ localDir: await apps.getStorageDir(app), remoteDir: 'data' }] : []); progressCallback({ message: `Uploading app snapshot ${app.fqdn}`}); diff --git a/src/ldap.js b/src/ldap.js index 9de4d4391..5c1fd4421 100644 --- a/src/ldap.js +++ b/src/ldap.js @@ -558,7 +558,7 @@ async function userSearchSftp(req, res, next) { const obj = { dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(), attributes: { - homeDirectory: app.dataDir ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js + homeDirectory: app.storageVolumeId ? `/mnt/app-${app.id}` : `/mnt/appsdata/${app.id}/data`, // see also sftp.js objectclass: ['user'], objectcategory: 'person', cn: user.id, diff --git a/src/routes/apps.js b/src/routes/apps.js index 92d91e228..d3efc2f72 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -35,7 +35,7 @@ exports = module.exports = { setMailbox, setInbox, setLocation, - setDataDir, + setStorage, setMounts, stop, @@ -432,13 +432,14 @@ async function setLocation(req, res, next) { next(new HttpSuccess(202, { taskId: result.taskId })); } -async function setDataDir(req, res, next) { +async function setStorage(req, res, next) { assert.strictEqual(typeof req.body, 'object'); assert.strictEqual(typeof req.app, 'object'); - if (req.body.dataDir !== null && typeof req.body.dataDir !== 'string') return next(new HttpError(400, 'dataDir must be a string')); + if (req.body.storageVolumeId !== null && typeof req.body.dataVolumeId !== 'string') return next(new HttpError(400, 'storageVolumeId must be a string')); + if (req.body.storageVolumePrefix !== null && typeof req.body.storageVolumePrefix !== 'string') return next(new HttpError(400, 'storageVolumePrefix must be a string')); - const [error, result] = await safe(apps.setDataDir(req.app, req.body.dataDir, AuditSource.fromRequest(req))); + const [error, result] = await safe(apps.setStorage(req.app, req.body.storageVolumeId, req.body.storageVolumePrefix, AuditSource.fromRequest(req))); if (error) return next(BoxError.toHttpError(error)); next(new HttpSuccess(202, { taskId: result.taskId })); diff --git a/src/server.js b/src/server.js index 46b2aa7fc..dd6106294 100644 --- a/src/server.js +++ b/src/server.js @@ -223,7 +223,7 @@ function initializeExpressSync() { router.post('/api/v1/apps/:id/configure/mailbox', json, token, routes.apps.load, authorizeAdmin, routes.apps.setMailbox); router.post('/api/v1/apps/:id/configure/inbox', json, token, routes.apps.load, authorizeAdmin, routes.apps.setInbox); router.post('/api/v1/apps/:id/configure/env', json, token, routes.apps.load, authorizeOperator, routes.apps.setEnvironment); - router.post('/api/v1/apps/:id/configure/data_dir', json, token, routes.apps.load, authorizeAdmin, routes.apps.setDataDir); + router.post('/api/v1/apps/:id/configure/storage', json, token, routes.apps.load, authorizeAdmin, routes.apps.setStorage); router.post('/api/v1/apps/:id/configure/location', json, token, routes.apps.load, authorizeAdmin, routes.apps.setLocation); router.post('/api/v1/apps/:id/configure/mounts', json, token, routes.apps.load, authorizeAdmin, routes.apps.setMounts); router.post('/api/v1/apps/:id/configure/crontab', json, token, routes.apps.load, authorizeOperator, routes.apps.setCrontab); diff --git a/src/services.js b/src/services.js index ae5afeb19..1c52118bc 100644 --- a/src/services.js +++ b/src/services.js @@ -858,7 +858,7 @@ async function setupLocalStorage(app, options) { debug('setupLocalStorage'); - const volumeDataDir = await apps.getDataDir(app, app.dataDir); + const volumeDataDir = await apps.getStorageDir(app); const [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {})); if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app storage data dir: ${error.message}`); @@ -870,7 +870,7 @@ async function clearLocalStorage(app, options) { debug('clearLocalStorage'); - const volumeDataDir = await apps.getDataDir(app, app.dataDir); + const volumeDataDir = await apps.getStorageDir(app); const [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, 'clear', volumeDataDir ], {}); if (error) throw new BoxError(BoxError.FS_ERROR, error); } @@ -881,7 +881,7 @@ async function teardownLocalStorage(app, options) { debug('teardownLocalStorage'); - const volumeDataDir = await apps.getDataDir(app, app.dataDir); + const volumeDataDir = await apps.getStorageDir(app); const [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, 'rmdir', volumeDataDir ], {}); if (error) throw new BoxError(BoxError.FS_ERROR, error); } diff --git a/src/sftp.js b/src/sftp.js index 72a195aca..4925074c1 100644 --- a/src/sftp.js +++ b/src/sftp.js @@ -62,9 +62,9 @@ async function start(existingInfra) { // custom app data directories const allApps = await apps.list(); for (const app of allApps) { - if (!app.manifest.addons['localstorage'] || !app.dataDir) continue; + if (!app.manifest.addons['localstorage'] || !app.storageVolumeId) continue; - const hostDir = await apps.getDataDir(app, app.dataDir), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp + const hostDir = await apps.getStorageDir(app), mountDir = `/mnt/app-${app.id}`; // see also sftp:userSearchSftp if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); diff --git a/src/system.js b/src/system.js index 6f95bf860..ddc336fae 100644 --- a/src/system.js +++ b/src/system.js @@ -41,10 +41,11 @@ async function getAppDisks(appsDataDisk) { const allApps = await apps.list(); for (const app of allApps) { - if (!app.dataDir) { + if (!app.storageVolumeId) { appDisks[app.id] = appsDataDisk; } else { - const [error, result] = await safe(df.file(app.dataDir)); + const dataDir = await apps.getStorageDir(app); + const [error, result] = await safe(df.file(dataDir)); appDisks[app.id] = error ? appsDataDisk : result.filesystem; // ignore any errors } }