diff --git a/src/boxerror.js b/src/boxerror.js index 3fa4ebe3f..238629e77 100644 --- a/src/boxerror.js +++ b/src/boxerror.js @@ -47,13 +47,14 @@ BoxError.DOCKER_ERROR = 'Docker Error'; BoxError.EXTERNAL_ERROR = 'External Error'; // use this for external API errors BoxError.FEATURE_DISABLED = 'Feature Disabled'; BoxError.FS_ERROR = 'FileSystem Error'; -BoxError.INACTIVE = 'Inactive'; +BoxError.INACTIVE = 'Inactive'; // service/volume/mount BoxError.INTERNAL_ERROR = 'Internal Error'; BoxError.INVALID_CREDENTIALS = 'Invalid Credentials'; BoxError.IPTABLES_ERROR = 'IPTables Error'; BoxError.LICENSE_ERROR = 'License Error'; BoxError.LOGROTATE_ERROR = 'Logrotate Error'; BoxError.MAIL_ERROR = 'Mail Error'; +BoxError.MOUNT_ERROR = 'Mount Error'; BoxError.NETWORK_ERROR = 'Network Error'; BoxError.NGINX_ERROR = 'Nginx Error'; BoxError.NOT_FOUND = 'Not found'; @@ -90,6 +91,7 @@ BoxError.toHttpError = function (error) { case BoxError.EXTERNAL_ERROR: case BoxError.NETWORK_ERROR: case BoxError.FS_ERROR: + case BoxError.MOUNT_ERROR: case BoxError.MAIL_ERROR: case BoxError.DOCKER_ERROR: case BoxError.ADDONS_ERROR: diff --git a/src/mounts.js b/src/mounts.js index 1399d8087..0918eb7f5 100644 --- a/src/mounts.js +++ b/src/mounts.js @@ -4,7 +4,8 @@ exports = module.exports = { writeMountFile, removeMountFile, validateMountOptions, - getStatus + getStatus, + waitForMount }; const assert = require('assert'), @@ -16,6 +17,7 @@ const assert = require('assert'), paths = require('./paths.js'), safe = require('safetydance'), shell = require('./shell.js'); +const promiseRetry = require('./promise-retry.js'); const ADD_MOUNT_CMD = path.join(__dirname, 'scripts/addmount.sh'); const RM_MOUNT_CMD = path.join(__dirname, 'scripts/rmmount.sh'); @@ -79,7 +81,7 @@ async function writeMountFile(volume) { what = mountOptions.diskPath; // like /dev/disk/by-uuid/uuid or /dev/disk/by-id/scsi-id options = 'discard,defaults,noatime'; break; - case 'sshfs': + case 'sshfs': { const keyFilePath = path.join(paths.SSHFS_KEYS_DIR, `id_rsa_${mountOptions.host}`); safe.fs.mkdirSync(paths.SSHFS_KEYS_DIR); @@ -90,6 +92,10 @@ async function writeMountFile(volume) { options = `defaults,allow_other,port=${mountOptions.port},IdentityFile=${keyFilePath},reconnect,uid=yellowtent,gid=yellowtent`; break; } + case 'noop': // volume provider + case 'mountpoint': // backup provider + return; + } const mountFileContents = ejs.render(SYSTEMD_MOUNT_EJS, { name, what, where: hostPath, options, type }); await shell.promises.sudo('generateMountFile', [ ADD_MOUNT_CMD, mountFileContents ], {}); @@ -116,7 +122,7 @@ async function getStatus(mountType, hostPath) { } let output = safe.child_process.execSync(`systemctl show --value -p ActiveState $(systemd-escape -p --suffix=mount ${hostPath})`, { encoding: 'utf8' }); - let state = output ? output.trim() : ''; + const state = output ? output.trim() : ''; let message; if (state !== 'active') { // find why it failed @@ -134,3 +140,17 @@ async function getStatus(mountType, hostPath) { return { state, message }; } + +async function waitForMount(mountType, hostPath, options) { + assert.strictEqual(typeof mountType, 'string'); + assert.strictEqual(typeof hostPath, 'string'); + assert.strictEqual(typeof options, 'object'); + + if (mountType === 'noop' || mountType === 'mountpoint') return; // noop is from volume provider and mountpoint is from backup provider + + await promiseRetry(options, async function () { + const status = await getStatus(mountType, hostPath); + if (status.state === 'active') return true; + throw new BoxError(BoxError.MOUNT_ERROR, `Mount is not active (${status.state}): ${status.message}`); + }); +} diff --git a/src/routes/volumes.js b/src/routes/volumes.js index b9bcb5b2f..40602764d 100644 --- a/src/routes/volumes.js +++ b/src/routes/volumes.js @@ -36,6 +36,8 @@ async function add(req, res, next) { if (typeof req.body.mountType !== 'string') return next(new HttpError(400, 'mountType must be a string')); if (typeof req.body.mountOptions !== 'object') return next(new HttpError(400, 'mountOptions must be a object')); + req.clearTimeout(); // waiting for mount can take time + const [error, id] = await safe(volumes.add(req.body, auditSource.fromRequest(req))); if (error) return next(BoxError.toHttpError(error)); next(new HttpSuccess(201, { id })); @@ -67,6 +69,8 @@ async function update(req, res, next) { if (typeof req.body.mountType !== 'string') return next(new HttpError(400, 'mountType must be a string')); if (typeof req.body.mountOptions !== 'object') return next(new HttpError(400, 'mountOptions must be a object')); + req.clearTimeout(); // waiting for mount can take time + const [error] = await safe(volumes.update(req.resource, req.body.mountType, req.body.mountOptions)); if (error) return next(BoxError.toHttpError(error)); next(new HttpSuccess(200, {})); diff --git a/src/volumes.js b/src/volumes.js index 3702f27d2..3d8f73363 100644 --- a/src/volumes.js +++ b/src/volumes.js @@ -69,32 +69,6 @@ function validateHostPath(hostPath, mountType) { return null; } -async function update(volume, mountType, mountOptions) { - assert.strictEqual(typeof volume, 'object'); - assert.strictEqual(typeof mountType, 'string'); - assert.strictEqual(typeof mountOptions, 'object'); - - let error = mounts.validateMountOptions(mountType, mountOptions); - if (error) throw error; - - if (mountType === 'noop') { - await mounts.removeMountFile(volume.hostPath); - } else { - await mounts.writeMountFile(Object.assign({}, volume, { mountType, mountOptions })); - } - - let result; - [error, result] = await safe(database.query('UPDATE volumes SET mountType=?, mountOptionsJson=? WHERE id=?', [ mountType, JSON.stringify(mountOptions), volume.id ])); - if (error) throw error; - if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Volume not found'); -} - -async function getStatus(volume) { - assert.strictEqual(typeof volume, 'object'); - - return await mounts.getStatus(volume.mountType, volume.hostPath); // { state, message } -} - async function add(volume, auditSource) { assert.strictEqual(typeof volume, 'object'); assert.strictEqual(typeof auditSource, 'object'); @@ -112,8 +86,15 @@ async function add(volume, auditSource) { const id = uuid.v4(); + // try mounting and unmount on fail + await mounts.writeMountFile(volume); + [error] = await safe(mounts.waitForMount(mountType, hostPath, { times: 20, interval: 500 })); // 10 seconds + if (error) { + await safe(mounts.removeMountFile(hostPath)); + throw error; + } + try { - if (mountType !== 'noop') await mounts.writeMountFile(volume); await database.query('INSERT INTO volumes (id, name, hostPath, mountType, mountOptionsJson) VALUES (?, ?, ?, ?, ?)', [ id, name, hostPath, mountType, JSON.stringify(mountOptions) ]); } catch (error) { if (error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('name') !== -1) throw new BoxError(BoxError.ALREADY_EXISTS, 'name already exists'); @@ -131,6 +112,36 @@ async function add(volume, auditSource) { return id; } +async function update(volume, mountType, mountOptions) { + assert.strictEqual(typeof volume, 'object'); + assert.strictEqual(typeof mountType, 'string'); + assert.strictEqual(typeof mountOptions, 'object'); + + let error = mounts.validateMountOptions(mountType, mountOptions); + if (error) throw error; + + if (mountType === 'noop') { + await mounts.removeMountFile(volume.hostPath); + } else { + await mounts.writeMountFile(Object.assign({}, volume, { mountType, mountOptions })); + + [error] = await safe(mounts.waitForMount(mountType, volume.hostPath, { times: 20, interval: 500 })); // 10 seconds + if (error) { + await safe(mounts.removeMountFile(volume.hostPath)); + throw error; + } + } + + const [, result] = await database.query('UPDATE volumes SET mountType=?, mountOptionsJson=? WHERE id=?', [ mountType, JSON.stringify(mountOptions), volume.id ]); + if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Volume not found'); +} + +async function getStatus(volume) { + assert.strictEqual(typeof volume, 'object'); + + return await mounts.getStatus(volume.mountType, volume.hostPath); // { state, message } +} + async function get(id) { assert.strictEqual(typeof id, 'string');