diff --git a/CHANGES b/CHANGES index 60d035e32..8742a6c53 100644 --- a/CHANGES +++ b/CHANGES @@ -1920,4 +1920,5 @@ * fix bug in disk usage sorting * mail: aliases can be across domains * mail: allow an external MX to be set +* volumes! diff --git a/migrations/20200423020749-volumes-create-table.js b/migrations/20200423020749-volumes-create-table.js new file mode 100644 index 000000000..530f4cc1e --- /dev/null +++ b/migrations/20200423020749-volumes-create-table.js @@ -0,0 +1,39 @@ +'use strict'; + +exports.up = function(db, callback) { + var cmd1 = 'CREATE TABLE volumes(' + + 'id VARCHAR(128) NOT NULL UNIQUE,' + + 'name VARCHAR(256) NOT NULL UNIQUE,' + + 'hostPath VARCHAR(1024) NOT NULL UNIQUE,' + + 'creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,' + + 'PRIMARY KEY (id)) CHARACTER SET utf8 COLLATE utf8_bin'; + + var cmd2 = 'CREATE TABLE appVolumes(' + + 'appId VARCHAR(128) NOT NULL,' + + 'volumeId VARCHAR(128) NOT NULL,' + + 'readOnly BOOLEAN DEFAULT 1,' + + 'FOREIGN KEY(appId) REFERENCES apps(id),' + + 'FOREIGN KEY(volumeId) REFERENCES volumes(id)) CHARACTER SET utf8 COLLATE utf8_bin;'; + + db.runSql(cmd1, function (error) { + if (error) console.error(error); + + db.runSql(cmd2, function (error) { + if (error) console.error(error); + + callback(error); + }); + }); +}; + +exports.down = function(db, callback) { + db.runSql('DROP TABLE appVolumes', function (error) { + if (error) console.error(error); + + db.runSql('DROP TABLE volumes', function (error) { + if (error) console.error(error); + callback(error); + }); + }); +}; + diff --git a/migrations/schema.sql b/migrations/schema.sql index e09d7c08d..c0d554f9a 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -233,4 +233,18 @@ CREATE TABLE IF NOT EXISTS appPasswords( PRIMARY KEY (id) ); +CREATE TABLE IF NOT EXISTS volumes( + id VARCHAR(128) NOT NULL UNIQUE, + name VARCHAR(256) NOT NULL UNIQUE, + hostPath VARCHAR(1024) NOT NULL UNIQUE, + creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS appVolumes( + appId VARCHAR(128) NOT NULL, + volumeId VARCHAR(128) NOT NULL, + FOREIGN KEY(appId) REFERENCES apps(id), + FOREIGN KEY(volumeId) REFERENCES volumes(id)); + CHARACTER SET utf8 COLLATE utf8_bin; diff --git a/setup/start.sh b/setup/start.sh index c0fcef06b..de558d4e0 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -20,6 +20,11 @@ readonly ubuntu_version=$(lsb_release -rs) cp -f "${script_dir}/../scripts/cloudron-support" /usr/bin/cloudron-support +# this needs to match the cloudron/base gid +if ! getent group media; then + addgroup --gid 500 --system media +fi + echo "==> Configuring docker" cp "${script_dir}/start/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app systemctl enable apparmor diff --git a/src/addons.js b/src/addons.js index 4d33f6300..8280b13d5 100644 --- a/src/addons.js +++ b/src/addons.js @@ -751,8 +751,8 @@ function setupLocalStorage(app, options, callback) { // reomve any existing volume in case it's bound with an old dataDir async.series([ - docker.removeVolume.bind(null, app, `${app.id}-localstorage`), - docker.createVolume.bind(null, app, `${app.id}-localstorage`, volumeDataDir) + docker.removeVolume.bind(null, `${app.id}-localstorage`), + docker.createVolume.bind(null, `${app.id}-localstorage`, volumeDataDir, { fqdn: app.fqdn, appId: app.id }) ], callback); } @@ -763,7 +763,7 @@ function clearLocalStorage(app, options, callback) { debugApp(app, 'clearLocalStorage'); - docker.clearVolume(app, `${app.id}-localstorage`, { removeDirectory: false }, callback); + docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: false }, callback); } function teardownLocalStorage(app, options, callback) { @@ -774,8 +774,8 @@ function teardownLocalStorage(app, options, callback) { debugApp(app, 'teardownLocalStorage'); async.series([ - docker.clearVolume.bind(null, app, `${app.id}-localstorage`, { removeDirectory: true }), - docker.removeVolume.bind(null, app, `${app.id}-localstorage`) + docker.clearVolume.bind(null, `${app.id}-localstorage`, { removeDirectory: true }), + docker.removeVolume.bind(null, `${app.id}-localstorage`) ], callback); } diff --git a/src/appdb.js b/src/appdb.js index a741af9e2..e3f0b0126 100644 --- a/src/appdb.js +++ b/src/appdb.js @@ -108,6 +108,10 @@ function postProcess(result) { if (envNames[i]) result.env[envNames[i]] = envValues[i]; } + let volumeIds = JSON.parse(result.volumeIds); + delete result.volumeIds; + result.volumeIds = volumeIds[0] === null ? [] : volumeIds; // NOTE: volumeIds is [null] when volumes of an app is empty + result.error = safe.JSON.parse(result.errorJson); delete result.errorJson; @@ -120,11 +124,13 @@ function get(id, callback) { database.query('SELECT ' + APPS_FIELDS_PREFIXED + ',' + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes, ' - + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues' + + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues, ' + + 'JSON_ARRAYAGG(appVolumes.volumeId) AS volumeIds ' + ' FROM apps' + ' LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId' + ' LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId' + ' LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?' + + ' LEFT OUTER JOIN appVolumes ON apps.id = appVolumes.appId' + ' WHERE apps.id = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, id ], function (error, result) { if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); @@ -147,11 +153,13 @@ function getByHttpPort(httpPort, callback) { database.query('SELECT ' + APPS_FIELDS_PREFIXED + ',' + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,' - + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues' + + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,' + + 'JSON_ARRAYAGG(appVolumes.volumeId) AS volumeIds ' + ' FROM apps' + ' LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId' + ' LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId' + ' LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?' + + ' LEFT OUTER JOIN appVolumes ON apps.id = appVolumes.appId' + ' WHERE httpPort = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, httpPort ], function (error, result) { if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); @@ -173,11 +181,13 @@ function getByContainerId(containerId, callback) { database.query('SELECT ' + APPS_FIELDS_PREFIXED + ',' + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,' - + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues' + + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,' + + 'JSON_ARRAYAGG(appVolumes.volumeId) AS volumeIds ' + ' FROM apps' + ' LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId' + ' LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId' + ' LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?' + + ' LEFT OUTER JOIN appVolumes ON apps.id = appVolumes.appId' + ' WHERE containerId = ? GROUP BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY, containerId ], function (error, result) { if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); @@ -198,11 +208,13 @@ function getAll(callback) { database.query('SELECT ' + APPS_FIELDS_PREFIXED + ',' + 'GROUP_CONCAT(CAST(appPortBindings.hostPort AS CHAR(6))) AS hostPorts, GROUP_CONCAT(appPortBindings.environmentVariable) AS environmentVariables, GROUP_CONCAT(appPortBindings.type) AS portTypes,' - + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues' + + 'JSON_ARRAYAGG(appEnvVars.name) AS envNames, JSON_ARRAYAGG(appEnvVars.value) AS envValues,' + + 'JSON_ARRAYAGG(appVolumes.volumeId) AS volumeIds ' + ' FROM apps' + ' LEFT OUTER JOIN appPortBindings ON apps.id = appPortBindings.appId' + ' LEFT OUTER JOIN appEnvVars ON apps.id = appEnvVars.appId' + ' LEFT OUTER JOIN subdomains ON apps.id = subdomains.appId AND subdomains.type = ?' + + ' LEFT OUTER JOIN appVolumes ON apps.id = appVolumes.appId' + ' GROUP BY apps.id ORDER BY apps.id', [ exports.SUBDOMAIN_TYPE_PRIMARY ], function (error, results) { if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); @@ -350,12 +362,13 @@ function del(id, callback) { { query: 'DELETE FROM appPortBindings WHERE appId = ?', args: [ id ] }, { query: 'DELETE FROM appEnvVars WHERE appId = ?', args: [ id ] }, { query: 'DELETE FROM appPasswords WHERE identifier = ?', args: [ id ] }, + { query: 'DELETE FROM appVolumes WHERE appId = ?', args: [ id ] }, { query: 'DELETE FROM apps WHERE id = ?', args: [ id ] } ]; database.transaction(queries, function (error, results) { if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); - if (results[4].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); + if (results[5].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); callback(null); }); @@ -425,12 +438,19 @@ function updateWithConstraints(id, app, constraints, callback) { }); } + if ('volumes' in app) { + queries.push({ query: 'DELETE FROM appVolumes WHERE appId = ?', args: [ id ]}); + app.volumes.forEach(function (v) { + queries.push({ query: 'INSERT INTO appVolumes (appId, volumeId, readOnly) VALUES (?, ?, ?)', args: [ id, v.id, v.readOnly ]}); + }); + } + var fields = [ ], values = [ ]; for (var p in app) { if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig') { fields.push(`${p}Json = ?`); values.push(JSON.stringify(app[p])); - } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'env') { + } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'env' && p !== 'volumes') { fields.push(p + ' = ?'); values.push(app[p]); } diff --git a/src/apps.js b/src/apps.js index f137f1085..19a344ce3 100644 --- a/src/apps.js +++ b/src/apps.js @@ -20,6 +20,7 @@ exports = module.exports = { setTags: setTags, setMemoryLimit: setMemoryLimit, setCpuShares: setCpuShares, + setVolumes: setVolumes, setAutomaticBackup: setAutomaticBackup, setAutomaticUpdate: setAutomaticUpdate, setReverseProxyConfig: setReverseProxyConfig, @@ -404,7 +405,7 @@ function removeInternalFields(app) { 'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain', 'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags', - 'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir'); + 'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'volumeIds'); } // non-admins can only see these @@ -968,6 +969,29 @@ function setCpuShares(app, cpuShares, auditSource, callback) { }); } +function setVolumes(app, volumes, auditSource, callback) { + assert.strictEqual(typeof app, 'object'); + assert(Array.isArray(volumes)); + assert.strictEqual(typeof auditSource, 'object'); + assert.strictEqual(typeof callback, 'function'); + + const appId = app.id; + let error = checkAppState(app, exports.ISTATE_PENDING_RECREATE_CONTAINER); + if (error) return callback(error); + + const task = { + args: {}, + values: { volumes } + }; + addTask(appId, exports.ISTATE_PENDING_RECREATE_CONTAINER, task, function (error, result) { + if (error) return callback(error); + + eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, volumes, taskId: result.taskId }); + + callback(null, { taskId: result.taskId }); + }); +} + function setEnvironment(app, env, auditSource, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof env, 'object'); diff --git a/src/docker.js b/src/docker.js index 84206a9bb..456610911 100644 --- a/src/docker.js +++ b/src/docker.js @@ -47,7 +47,7 @@ var addons = require('./addons.js'), shell = require('./shell.js'), safe = require('safetydance'), util = require('util'), - _ = require('underscore'); + volumes = require('./volumes.js'); const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'), MKDIRVOLUME_CMD = path.join(__dirname, 'scripts/mkdirvolume.sh'); @@ -188,11 +188,34 @@ function downloadImage(manifest, callback) { }, callback); } -function createSubcontainer(app, name, cmd, options, callback) { +function getEnvAndMounts(app, callback) { + addons.getEnvironment(app, function (error, addonEnv) { + if (error) return callback(error); + + let mounts = addons.getMountsSync(app, app.manifest.addons); + + volumes.getAppMounts(app.id, function (error, volumes) { + if (error) return callback(error); + + volumes.forEach(function (volume) { + mounts.push({ + Target: `/media/${volume.name}`, // mnt is for temporary mounts. media is for removable mounts + Source: volume.id, + Type: 'volume', + ReadOnly: volume.readOnly + }); + }); + + callback(null, addonEnv, mounts); + }); + + }); +} + +function createSubcontainer(app, name, cmd, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof name, 'string'); assert(!cmd || util.isArray(cmd)); - assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); let isAppContainer = !cmd; // non app-containers are like scheduler and exec (terminal) containers @@ -249,7 +272,7 @@ function createSubcontainer(app, name, cmd, options, callback) { // if required, we can make this a manifest and runtime argument later if (!isAppContainer) memoryLimit *= 2; - addons.getEnvironment(app, function (error, addonEnv) { + getEnvAndMounts(app, function (error, addonEnv, mounts) { if (error) return callback(error); // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail @@ -276,7 +299,7 @@ function createSubcontainer(app, name, cmd, options, callback) { 'isCloudronManaged': String(true) }, HostConfig: { - Mounts: addons.getMountsSync(app, app.manifest.addons), + Mounts: mounts, LogConfig: { Type: 'syslog', Config: { @@ -317,8 +340,6 @@ function createSubcontainer(app, name, cmd, options, callback) { ]; } - containerOptions = _.extend(containerOptions, options); - debugApp(app, 'Creating container for %s', app.manifest.dockerImage); gConnection.createContainer(containerOptions, function (error, container) { @@ -330,7 +351,7 @@ function createSubcontainer(app, name, cmd, options, callback) { } function createContainer(app, callback) { - createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback); + createSubcontainer(app, app.id /* name */, null /* cmd */, callback); } function startContainer(containerId, callback) { @@ -573,10 +594,10 @@ function memoryUsage(containerId, callback) { }); } -function createVolume(app, name, volumeDataDir, callback) { - assert.strictEqual(typeof app, 'object'); +function createVolume(name, volumeDataDir, labels, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof volumeDataDir, 'string'); + assert.strictEqual(typeof labels, 'object'); assert.strictEqual(typeof callback, 'function'); const volumeOptions = { @@ -587,10 +608,7 @@ function createVolume(app, name, volumeDataDir, callback) { device: volumeDataDir, o: 'bind' }, - Labels: { - 'fqdn': app.fqdn, - 'appId': app.id - }, + Labels: labels }; // requires sudo because the path can be outside appsdata @@ -600,13 +618,12 @@ function createVolume(app, name, volumeDataDir, callback) { gConnection.createVolume(volumeOptions, function (error) { if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - callback(); + callback(null); }); }); } -function clearVolume(app, name, options, callback) { - assert.strictEqual(typeof app, 'object'); +function clearVolume(name, options, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof options, 'object'); assert.strictEqual(typeof callback, 'function'); @@ -626,14 +643,13 @@ function clearVolume(app, name, options, callback) { } // this only removes the volume and not the data -function removeVolume(app, name, callback) { - assert.strictEqual(typeof app, 'object'); +function removeVolume(name, callback) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof callback, 'function'); let volume = gConnection.getVolume(name); volume.remove(function (error) { - if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`)); + if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error removing volume: ${error.message}`)); callback(); }); diff --git a/src/eventlog.js b/src/eventlog.js index 179c18253..18758fcce 100644 --- a/src/eventlog.js +++ b/src/eventlog.js @@ -60,6 +60,9 @@ exports = module.exports = { ACTION_USER_UPDATE: 'user.update', ACTION_USER_TRANSFER: 'user.transfer', + ACTION_VOLUME_ADD: 'volume.add', + ACTION_VOLUME_REMOVE: 'volume.remove', + ACTION_DYNDNS_UPDATE: 'dyndns.update', ACTION_SUPPORT_TICKET: 'support.ticket', diff --git a/src/routes/apps.js b/src/routes/apps.js index 313e2eb70..b7e6549d2 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -21,6 +21,7 @@ exports = module.exports = { setIcon: setIcon, setMemoryLimit: setMemoryLimit, setCpuShares: setCpuShares, + setVolumes: setVolumes, setAutomaticBackup: setAutomaticBackup, setAutomaticUpdate: setAutomaticUpdate, setReverseProxyConfig: setReverseProxyConfig, @@ -238,6 +239,24 @@ function setCpuShares(req, res, next) { }); } +function setVolumes(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + assert.strictEqual(typeof req.resource, 'object'); + + if (!Array.isArray(req.body.volumes)) return next(new HttpError(400, 'volumes should be an array')); + for (let v of req.body.volumes) { + if (!v || typeof v !== 'object') return next(new HttpError(400, 'volume must be a {id, readOnly}')); + if (typeof v.id !== 'string') return next(new HttpError(400, 'id must be a string')); + if (typeof v.readOnly !== 'boolean') return next(new HttpError(400, 'readOnly must be a boolean')); + } + + apps.setVolumes(req.resource, req.body.volumes, auditSource.fromRequest(req), function (error, result) { + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(202, { taskId: result.taskId })); + }); +} + function setAutomaticBackup(req, res, next) { assert.strictEqual(typeof req.body, 'object'); assert.strictEqual(typeof req.resource, 'object'); diff --git a/src/routes/index.js b/src/routes/index.js index 5aed8a521..03c5a18ed 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -22,5 +22,6 @@ exports = module.exports = { support: require('./support.js'), tasks: require('./tasks.js'), tokens: require('./tokens.js'), - users: require('./users.js') + users: require('./users.js'), + volumes: require('./volumes.js') }; diff --git a/src/routes/test/apps-test.js b/src/routes/test/apps-test.js index 2199a773c..ea96e37e7 100644 --- a/src/routes/test/apps-test.js +++ b/src/routes/test/apps-test.js @@ -1479,6 +1479,52 @@ describe('App API', function () { }); }); + describe('volumes', function () { + let volumeId; + + it('can add volume', function (done) { + superagent.post(SERVER_URL + '/api/v1/volumes') + .query({ access_token: token }) + .send({ name: 'videos', hostPath: '/media/cloudron-test' }) + .end(function (err, res) { + expect(res.statusCode).to.equal(201); + expect(res.body.id).to.be.a('string'); + volumeId = res.body.id; + done(); + }); + }); + + it('can set volume', function (done) { + superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure/volumes') + .query({ access_token: token }) + .send({ volumes: [ {id: volumeId, readOnly: false} ] }) + .end(function (err, res) { + expect(res.statusCode).to.equal(202); + taskId = res.body.taskId; + done(); + }); + }); + + it('wait for task', function (done) { + waitForTask(taskId, done); + }); + + it('did set volume', function (done) { + apps.get(APP_ID, function (error, appEntry) { + if (error) return done(error); + + docker.getContainer(appEntry.containerId).inspect(function (error, data) { + expect(error).to.not.be.ok(); + expect(data.Mounts.filter(function (mount) { return mount.Destination === '/media/videos'; })[0].Type).to.eql('volume'); + expect(data.Mounts.filter(function (mount) { return mount.Destination === '/media/videos'; })[0].Name).to.eql(volumeId); + expect(data.Mounts.filter(function (mount) { return mount.Destination === '/media/videos'; })[0].RW).to.eql(true); + + done(); + }); + }); + }); + }); + describe('start/stop', function () { it('non admin cannot stop app', function (done) { superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/stop') diff --git a/src/routes/test/volumes-test.js b/src/routes/test/volumes-test.js new file mode 100644 index 000000000..518ec05d7 --- /dev/null +++ b/src/routes/test/volumes-test.js @@ -0,0 +1,155 @@ +'use strict'; + +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +var async = require('async'), + constants = require('../../constants.js'), + database = require('../../database.js'), + expect = require('expect.js'), + server = require('../../server.js'), + superagent = require('superagent'); + +var SERVER_URL = 'http://localhost:' + constants.PORT; + +var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com'; +var token = null; + +function setup(done) { + async.series([ + server.start.bind(null), + database._clear.bind(null), + + function createAdmin(callback) { + superagent.post(SERVER_URL + '/api/v1/cloudron/activate') + .query({ setupToken: 'somesetuptoken' }) + .send({ username: USERNAME, password: PASSWORD, email: EMAIL }) + .end(function (error, result) { + expect(result).to.be.ok(); + expect(result.statusCode).to.eql(201); + + // stash token for further use + token = result.body.token; + + callback(); + }); + } + ], done); +} + +function cleanup(done) { + database._clear(function (error) { + expect(!error).to.be.ok(); + + server.stop(done); + }); +} + +describe('Volume API', function () { + before(setup); + after(cleanup); + let volumeId; + + describe('add', function () { + it('bad name', function (done) { + superagent.post(SERVER_URL + '/api/v1/volumes') + .query({ access_token: token }) + .send({ name: 'som*', hostPath: '/media/cloudron-test' }) + .end(function (err, res) { + expect(res.statusCode).to.equal(400); + expect(res.body.message).to.contain('name'); + done(); + }); + }); + + it('bad path', function (done) { + superagent.post(SERVER_URL + '/api/v1/volumes') + .query({ access_token: token }) + .send({ name: 'videos', hostPath: '/tmp/cloudron-test' }) + .end(function (err, res) { + expect(res.statusCode).to.equal(400); + expect(res.body.message).to.contain('hostPath'); + done(); + }); + }); + + it('can add volume', function (done) { + superagent.post(SERVER_URL + '/api/v1/volumes') + .query({ access_token: token }) + .send({ name: 'videos', hostPath: '/media/cloudron-test' }) + .end(function (err, res) { + expect(res.statusCode).to.equal(201); + expect(res.body.id).to.be.a('string'); + volumeId = res.body.id; + done(); + }); + }); + + it('cannot add conflicting path', function (done) { + superagent.post(SERVER_URL + '/api/v1/volumes') + .query({ access_token: token }) + .send({ name: 'videos2', hostPath: '/media/cloudron-test' }) + .end(function (err, res) { + expect(res.statusCode).to.equal(409); + done(); + }); + }); + }); + + describe('get', function () { + it('cannot get random volume', function (done) { + superagent.get(SERVER_URL + '/api/v1/volumes/someid') + .query({ access_token: token }) + .end(function (err, res) { + expect(res.statusCode).to.equal(404); + done(); + }); + }); + + it('can get valid volume', function (done) { + superagent.get(SERVER_URL + `/api/v1/volumes/${volumeId}`) + .query({ access_token: token }) + .end(function (err, res) { + expect(res.statusCode).to.equal(200); + expect(res.body.hostPath).to.be('/media/cloudron-test'); + done(); + }); + }); + }); + + describe('list', function () { + it('can list', function (done) { + superagent.get(SERVER_URL + '/api/v1/volumes') + .query({ access_token: token }) + .end(function (err, res) { + expect(res.statusCode).to.equal(200); + expect(res.body.volumes.length).to.be(1); + expect(res.body.volumes[0].name).to.be('videos'); + expect(res.body.volumes[0].hostPath).to.be('/media/cloudron-test'); + done(); + }); + }); + }); + + describe('del', function () { + it('cannot del random volume', function (done) { + superagent.del(SERVER_URL + '/api/v1/volumes/someid') + .query({ access_token: token }) + .end(function (err, res) { + expect(res.statusCode).to.equal(404); + done(); + }); + }); + + it('can del valid volume', function (done) { + superagent.del(SERVER_URL + `/api/v1/volumes/${volumeId}`) + .query({ access_token: token }) + .end(function (err, res) { + expect(res.statusCode).to.equal(204); + done(); + }); + }); + }); +}); diff --git a/src/routes/volumes.js b/src/routes/volumes.js new file mode 100644 index 000000000..4fc05e599 --- /dev/null +++ b/src/routes/volumes.js @@ -0,0 +1,56 @@ +'use strict'; + +exports = module.exports = { + add, + get, + del, + list +}; + +const assert = require('assert'), + auditSource = require('../auditsource.js'), + BoxError = require('../boxerror.js'), + volumes = require('../volumes.js'), + HttpError = require('connect-lastmile').HttpError, + HttpSuccess = require('connect-lastmile').HttpSuccess; + +function add(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + + if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string')); + if (typeof req.body.hostPath !== 'string') return next(new HttpError(400, 'hostPath must be a string')); + + volumes.add(req.body.name, req.body.hostPath, auditSource.fromRequest(req), function (error, id) { + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(201, { id })); + }); +} + +function get(req, res, next) { + assert.strictEqual(typeof req.params.id, 'string'); + + volumes.get(req.params.id, function (error, result) { + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(200, result)); + }); +} + +function del(req, res, next) { + assert.strictEqual(typeof req.params.id, 'string'); + + volumes.del(req.params.id, auditSource.fromRequest(req), function (error) { + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(204)); + }); +} + +function list(req, res, next) { + volumes.list(function (error, result) { + if (error) return next(new HttpError(500, error)); + + next(new HttpSuccess(200, { volumes: result })); + }); +} diff --git a/src/scheduler.js b/src/scheduler.js index f1a83bb0d..c40caf5d0 100644 --- a/src/scheduler.js +++ b/src/scheduler.js @@ -173,7 +173,7 @@ function runTask(appId, taskName, callback) { debug(`runTask: starting task ${taskName} of app ${app.fqdn} with cmd ${cmd}`); // NOTE: if you change container name here, fix addons.js to return correct container names - docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error, container) { + docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], function (error, container) { if (error) return callback(error); docker.startContainer(container.id, callback); diff --git a/src/server.js b/src/server.js index b1b8c71ef..a3994d87f 100644 --- a/src/server.js +++ b/src/server.js @@ -212,6 +212,7 @@ function initializeExpressSync() { router.post('/api/v1/apps/:id/configure/env', token, authorizeAdmin, routes.apps.load, routes.apps.setEnvironment); router.post('/api/v1/apps/:id/configure/data_dir', token, authorizeAdmin, routes.apps.load, routes.apps.setDataDir); router.post('/api/v1/apps/:id/configure/location', token, authorizeAdmin, routes.apps.load, routes.apps.setLocation); + router.post('/api/v1/apps/:id/configure/volumes', token, authorizeAdmin, routes.apps.load, routes.apps.setVolumes); router.post('/api/v1/apps/:id/repair', token, authorizeAdmin, routes.apps.load, routes.apps.repair); router.post('/api/v1/apps/:id/update', token, authorizeAdmin, routes.apps.load, routes.apps.update); @@ -287,6 +288,12 @@ function initializeExpressSync() { router.del ('/api/v1/domains/:domain', token, authorizeAdmin, routes.domains.del); router.get ('/api/v1/domains/:domain/dns_check', token, authorizeAdmin, routes.domains.checkDnsRecords); + // volume routes + router.post('/api/v1/volumes', token, authorizeAdmin, routes.volumes.add); + router.get ('/api/v1/volumes/:id', token, authorizeAdmin, routes.volumes.get); + router.del ('/api/v1/volumes/:id', token, authorizeAdmin, routes.volumes.del); + router.get ('/api/v1/volumes', token, authorizeAdmin, routes.volumes.list); + // addon routes router.get ('/api/v1/services', token, authorizeAdmin, routes.services.getAll); router.get ('/api/v1/services/:service', token, authorizeAdmin, routes.services.get); diff --git a/src/test/database-test.js b/src/test/database-test.js index 4f4e14142..224ed0434 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -23,6 +23,7 @@ var appdb = require('../appdb.js'), taskdb = require('../taskdb.js'), tokendb = require('../tokendb.js'), userdb = require('../userdb.js'), + volumedb = require('../volumedb.js'), _ = require('underscore'); var USER_0 = { @@ -418,7 +419,8 @@ describe('database', function () { dataDir: null, tags: [], label: null, - taskId: null + taskId: null, + volumeIds: [] }; it('cannot delete referenced domain', function (done) { @@ -606,7 +608,7 @@ describe('database', function () { }); it('can get all admins', function (done) { - userdb.getByRole('owner', function (error, all) { + userdb.getByRole('owner', function (error) { expect(error).to.be.ok(); expect(error.reason).to.be(BoxError.NOT_FOUND); done(); @@ -889,7 +891,8 @@ describe('database', function () { dataDir: null, tags: [], label: null, - taskId: null + taskId: null, + volumeIds: [] }; var APP_1 = { @@ -920,7 +923,8 @@ describe('database', function () { dataDir: null, tags: [], label: null, - taskId: null + taskId: null, + volumeIds: [] }; before(function (done) { @@ -1963,4 +1967,110 @@ describe('database', function () { }); }); }); + + describe('volumes', function () { + before(function (done) { + done(); + }); + + after(function (done) { + database._clear(done); + }); + + it('can add volume', function (done) { + volumedb.add('id1', 'myvolume', '/tmp/foo', function (error) { + expect(error).to.be(null); + + done(); + }); + }); + + it('cannot add conflicting name', function (done) { + volumedb.add('someotherid', 'myvolume', '/tmp/bar', function (error) { + expect(error.reason).to.be(BoxError.ALREADY_EXISTS); + + done(); + }); + }); + + it('cannot get random id', function (done) { + volumedb.get('randomid', function (error) { + expect(error.reason).to.be(BoxError.NOT_FOUND); + + done(); + }); + }); + + it('can get existing id', function (done) { + volumedb.get('id1', function (error, result) { + expect(error).to.be(null); + expect(result.name).to.be('myvolume'); + expect(result.hostPath).to.be('/tmp/foo'); + + done(); + }); + }); + + it('cannot update random id', function (done) { + volumedb.update('randomid', { name: 'theirvolume', hostPath: '/tmp/bar' }, function (error) { + expect(error.reason).to.be(BoxError.NOT_FOUND); + + done(); + }); + }); + + it('can add another volume', function (done) { + volumedb.add('id2', 'myvolume2', '/tmp/foo2', function (error) { + expect(error).to.be(null); + + done(); + }); + }); + + it('cannot update to existing path', function (done) { + volumedb.update('id1', { name: 'myvolume2', hostPath: '/tmp/foo' }, function (error) { + expect(error.reason).to.be(BoxError.ALREADY_EXISTS); + + done(); + }); + }); + + it('can update existing id', function (done) { + volumedb.update('id1', { hostPath: '/tmp/bar' }, function (error) { + expect(error).to.be(null); + + done(); + }); + }); + + it('can list', function (done) { + volumedb.list(function (error, result) { + expect(error).to.be(null); + expect(result.length).to.be(2); + expect(result[0].name).to.be('myvolume'); + expect(result[0].hostPath).to.be('/tmp/bar'); + + expect(result[1].name).to.be('myvolume2'); + expect(result[1].hostPath).to.be('/tmp/foo2'); + done(); + }); + }); + + + it('cannot delete random id', function (done) { + volumedb.del('randomid', function (error) { + expect(error.reason).to.be(BoxError.NOT_FOUND); + + done(); + }); + }); + + it('can delete existing id', function (done) { + volumedb.del('id1', function (error) { + expect(error).to.be(null); + + done(); + }); + }); + }); }); diff --git a/src/test/volumes-test.js b/src/test/volumes-test.js new file mode 100644 index 000000000..ee51e0874 --- /dev/null +++ b/src/test/volumes-test.js @@ -0,0 +1,92 @@ +/* jslint node:true */ +/* global it:false */ +/* global describe:false */ +/* global before:false */ +/* global after:false */ + +'use strict'; + +let async = require('async'), + BoxError = require('../boxerror.js'), + database = require('../database.js'), + expect = require('expect.js'), + volumes = require('../volumes.js'); + +const AUDIT_SOURCE = { ip: '1.2.3.4' }; + +function setup(done) { + async.series([ + database.initialize, + database._clear, + ], done); +} + +function cleanup(done) { + async.series([ + database._clear, + database.uninitialize + ], done); +} + +describe('volumes', function () { + before(setup); + after(cleanup); + + let volumeId; + + describe('add', function () { + it('cannot add invalid name', function (done) { + volumes.add('fan&', '/tmp/foo', AUDIT_SOURCE, function (error) { + expect(error.reason).to.be(BoxError.BAD_FIELD); + done(); + }); + }); + + it('cannot add invalid path', function (done) { + volumes.add('fan', '/tmp/foo', AUDIT_SOURCE, function (error) { + expect(error.reason).to.be(BoxError.BAD_FIELD); + done(); + }); + }); + + it('can add valid volume', function (done) { + volumes.add('cloudron-test', '/media/cloudron-test', AUDIT_SOURCE, function (error, result) { + expect(error).to.be(null); + expect(result).to.be.a('string'); + + volumeId = result; + + done(); + }); + }); + }); + + describe('list', function () { + it('can list volumes', function (done) { + volumes.list(function (error, results) { + expect(error).to.be(null); + expect(results.length).to.be(1); + expect(results[0].id).to.be(volumeId); + expect(results[0].name).to.be('cloudron-test'); + expect(results[0].hostPath).to.be('/media/cloudron-test'); + done(); + }); + }); + }); + + describe('del', function () { + it('cannot remove random volume', function (done) { + volumes.del('random', AUDIT_SOURCE, function (error) { + expect(error.reason).to.be(BoxError.NOT_FOUND); + done(); + }); + }); + + it('cannot remove existing volume', function (done) { + volumes.del(volumeId, AUDIT_SOURCE, function (error) { + expect(error).to.be(null); + done(); + }); + }); + }); +}); \ No newline at end of file diff --git a/src/volumedb.js b/src/volumedb.js new file mode 100644 index 000000000..a8b69470a --- /dev/null +++ b/src/volumedb.js @@ -0,0 +1,111 @@ +/* jslint node:true */ + +'use strict'; + +exports = module.exports = { + add, + get, + list, + update, + del, + clear, + getAppMounts +}; + +const assert = require('assert'), + BoxError = require('./boxerror.js'), + database = require('./database.js'); + +const VOLUMES_FIELDS = [ 'id', 'name', 'hostPath', 'creationTime' ].join(','); +const APP_VOLUMES_FIELDS = [ 'appId', 'volumeId', 'readOnly' ].join(','); + +function get(id, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof callback, 'function'); + + database.query(`SELECT ${VOLUMES_FIELDS} FROM volumes WHERE id=?`, [ id ], function (error, result) { + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Volume not found')); + + callback(null, result[0]); + }); +} + +function list(callback) { + database.query(`SELECT ${VOLUMES_FIELDS} FROM volumes ORDER BY name`, function (error, results) { + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + + callback(null, results); + }); +} + +function add(id, name, hostPath, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof name, 'string'); + assert.strictEqual(typeof hostPath, 'string'); + assert.strictEqual(typeof callback, 'function'); + + database.query('INSERT INTO volumes (id, name, hostPath) VALUES (?, ?, ?)', [ id, name, hostPath ], function (error) { + if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('name') !== -1) return callback(new BoxError(BoxError.ALREADY_EXISTS, 'name already exists')); + if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('hostPath') !== -1) return callback(new BoxError(BoxError.ALREADY_EXISTS, 'hostPath already exists')); + if (error && error.code === 'ER_DUP_ENTRY' && error.sqlMessage.indexOf('PRIMARY') !== -1) return callback(new BoxError(BoxError.ALREADY_EXISTS, 'id already exists')); + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + + callback(null); + }); +} + +function update(id, data, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof data, 'object'); + assert.strictEqual(typeof callback, 'function'); + + var args = [ ], fields = [ ]; + for (var k in data) { + fields.push(k + ' = ?'); + args.push(data[k]); + } + args.push(id); + + database.query('UPDATE volumes SET ' + fields.join(', ') + ' WHERE id=?', args, function (error, result) { + if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message)); + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); + + callback(null); + }); +} + +function del(id, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof callback, 'function'); + + database.query('DELETE FROM volumes WHERE id=?', [ id ], function (error, result) { + if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.CONFLICT, 'Volume is in use')); + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + if (result.affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Volume not found')); + + callback(null); + }); +} + +function clear(callback) { + database.query('DELETE FROM volumes', function (error) { + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + + callback(error); + }); +} + +function getAppMounts(appId, callback) { + assert.strictEqual(typeof appId, 'string'); + assert.strictEqual(typeof callback, 'function'); + + database.query(`SELECT ${APP_VOLUMES_FIELDS},${VOLUMES_FIELDS} FROM volumes INNER JOIN appVolumes ON volumes.id = appVolumes.volumeId WHERE appId=?`, [ appId ], function (error, result) { + if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error)); + + result.forEach(r => r.readOnly = !!r.readOnly); // make it bool + + callback(null, result); + }); +} \ No newline at end of file diff --git a/src/volumes.js b/src/volumes.js new file mode 100644 index 000000000..e3449ef65 --- /dev/null +++ b/src/volumes.js @@ -0,0 +1,107 @@ +'use strict'; + +exports = module.exports = { + add, + get, + del, + list, + // update, // volume cannot be changed because docker cannot change underlying volume paths (what if it's mounted) + getAppMounts +}; + +const assert = require('assert'), + BoxError = require('./boxerror.js'), + docker = require('./docker.js'), + eventlog = require('./eventlog.js'), + uuid = require('uuid'), + volumedb = require('./volumedb.js'); + +function validateName(name) { + assert.strictEqual(typeof name, 'string'); + + // just have friendly characters under /media + if (!/^[-0-9a-zA-Z_@$=#.%+]+$/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'Invalid name'); + + return null; +} + +function validateHostPath(hostPath) { + assert.strictEqual(typeof hostPath, 'string'); + + if (!hostPath.startsWith('/mnt') && !hostPath.startsWith('/media')) return new BoxError(BoxError.BAD_FIELD, 'hostPath must be in /mnt or /media'); + + return null; +} + +function add(name, hostPath, auditSource, callback) { + assert.strictEqual(typeof name, 'string'); + assert.strictEqual(typeof hostPath, 'string'); + assert.strictEqual(typeof auditSource, 'object'); + assert.strictEqual(typeof callback, 'function'); + + let error = validateName(name); + if (error) return callback(error); + + error = validateHostPath(hostPath); + if (error) return callback(error); + + const id = uuid() + '-media'; + + docker.createVolume(id, hostPath, { name }, function (error) { + if (error) return callback(error); + + volumedb.add(id, name, hostPath, function (error) { + if (error) return callback(error); + + eventlog.add(eventlog.ACTION_VOLUME_ADD, auditSource, { id, name, hostPath }); + + callback(null, id); + }); + }); +} + +function get(id, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof callback, 'function'); + + volumedb.get(id, function (error, result) { + if (error) return callback(error); + + callback(null, result); + }); +} + +function list(callback) { + assert.strictEqual(typeof callback, 'function'); + + volumedb.list(function (error, result) { + if (error) return callback(error); + + return callback(null, result); + }); +} + +function del(id, auditSource, callback) { + assert.strictEqual(typeof id, 'string'); + assert.strictEqual(typeof auditSource, 'object'); + assert.strictEqual(typeof callback, 'function'); + + docker.removeVolume(id, function (error) { + if (error) return callback(error); + + volumedb.del(id, function (error) { + if (error) return callback(error); + + eventlog.add(eventlog.ACTION_VOLUME_REMOVE, auditSource, { id }); + + return callback(null); + }); + }); +} + +function getAppMounts(appId, callback) { + assert.strictEqual(typeof appId, 'string'); + assert.strictEqual(typeof callback, 'function'); + + volumedb.getAppMounts(appId, callback); +}