From 8a17e13ec4fe6725cc34ca8812c0dbe9746d90ec Mon Sep 17 00:00:00 2001 From: Girish Ramakrishnan Date: Tue, 22 Dec 2020 17:19:26 -0800 Subject: [PATCH] automate wellknown setup the main reason this is under app and not domain is because it let's the user know that an app has to be installed for the whole thing to work. part of #703 --- CHANGES | 1 + .../20201223014453-apps-add-wellKnownJson.js | 46 +++++++++++++++++++ migrations/schema.sql | 1 + src/appdb.js | 5 +- src/apps.js | 19 +++++++- src/nginxconfig.ejs | 6 +-- src/routes/apps.js | 18 ++++++++ src/routes/index.js | 3 +- src/routes/wellknown.js | 21 +++++++++ src/server.js | 5 ++ 10 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 migrations/20201223014453-apps-add-wellKnownJson.js create mode 100644 src/routes/wellknown.js diff --git a/CHANGES b/CHANGES index ad98dd27d..552954bb8 100644 --- a/CHANGES +++ b/CHANGES @@ -2170,4 +2170,5 @@ * persist update indicator across restarts * cloudron-setup: add --generate-setup-token * dashboard: pass accessToken query param to automatically login +* wellknown: add a way to set well known docs diff --git a/migrations/20201223014453-apps-add-wellKnownJson.js b/migrations/20201223014453-apps-add-wellKnownJson.js new file mode 100644 index 000000000..300330ea8 --- /dev/null +++ b/migrations/20201223014453-apps-add-wellKnownJson.js @@ -0,0 +1,46 @@ +'use strict'; + +const async = require('async'), + safe = require('safetydance'); + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE apps ADD COLUMN wellKnownJson TEXT', function (error) { + if (error) return callback(error); + + // keep the paths around, so that we don't need to trigger a re-configure. the old nginx config will use the paths + // the new one will proxy calls to the box code + const WELLKNOWN_DIR = '/home/yellowtent/boxdata/well-known'; + const output = safe.child_process.execSync('find . -type f -printf "%P"', { cwd: WELLKNOWN_DIR, encoding: 'utf8' }); + if (!output) return callback(); + const paths = output.trim().split('\n'); + if (paths.length === 0) return callback(); // user didn't configure any well-known + + let wellKnown = {}; + for (let path of paths) { + const fqdn = path.split('/', 1)[0]; + const loc = path.slice(fqdn.length+1); + const doc = safe.fs.readFileSync(`${WELLKNOWN_DIR}/${path}`, { encoding: 'utf8' }); + if (!doc) continue; + + wellKnown[fqdn] = {}; + wellKnown[fqdn][loc] = doc; + } + + async.eachSeries(Object.keys(wellKnown), function (fqdn, iteratorDone) { + db.runSql('SELECT appId FROM subdomains WHERE domain=? AND subdomain=?', [ fqdn, '' ], function (error, result) { + if (error || result.length === 0) return iteratorDone(); // there could be no corresponding app + + db.runSql('UPDATE apps SET wellKnownJson=? WHERE id=?', [ JSON.stringify(wellKnown), result[0].appId ], iteratorDone); + }); + }, function (error) { + callback(error); + }); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE apps DROP COLUMN wellknownJson', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/migrations/schema.sql b/migrations/schema.sql index c5367ff80..d9c611378 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -86,6 +86,7 @@ CREATE TABLE IF NOT EXISTS apps( taskId INTEGER, // current task errorJson TEXT, servicesConfigJson TEXT, // app services configuration + wellKnownJson TEXT, containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair' FOREIGN KEY(mailboxDomain) REFERENCES domains(domain), diff --git a/src/appdb.js b/src/appdb.js index 6352491f4..7fa135d70 100644 --- a/src/appdb.js +++ b/src/appdb.js @@ -41,7 +41,7 @@ var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationSta 'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares', 'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', - 'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate', + 'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate', 'apps.wellKnownJson', 'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(','); var PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(','); @@ -118,6 +118,9 @@ function postProcess(result) { result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty + result.wellKnown = safe.JSON.parse(result.wellKnownJson); + delete result.wellKnownJson; + result.error = safe.JSON.parse(result.errorJson); delete result.errorJson; diff --git a/src/apps.js b/src/apps.js index ea03b0e02..1ca43bdf2 100644 --- a/src/apps.js +++ b/src/apps.js @@ -20,6 +20,7 @@ exports = module.exports = { setMemoryLimit, setCpuShares, setMounts, + setWellKnown, setAutomaticBackup, setAutomaticUpdate, setReverseProxyConfig, @@ -405,7 +406,7 @@ function removeInternalFields(app) { 'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain', 'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags', - 'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts'); + 'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts', 'wellKnown'); } // non-admins can only see these @@ -982,6 +983,22 @@ function setMounts(app, mounts, auditSource, callback) { }); } +function setWellKnown(app, wellKnown, auditSource, callback) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof wellKnown, 'object'); + assert.strictEqual(typeof auditSource, 'object'); + assert.strictEqual(typeof callback, 'function'); + + const appId = app.id; + appdb.update(appId, { wellKnown }, function (error) { + if (error) return callback(error); + + eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, wellKnown }); + + callback(); + }); +} + function setEnvironment(app, env, auditSource, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof env, 'object'); diff --git a/src/nginxconfig.ejs b/src/nginxconfig.ejs index 40602957a..9cf5c2393 100644 --- a/src/nginxconfig.ejs +++ b/src/nginxconfig.ejs @@ -162,9 +162,9 @@ server { } # user defined .well-known resources - location ~ ^/.well-known/(.*)$ { - root /home/yellowtent/boxdata/well-known/$host; - try_files /$1 @wellknown-upstream; + location /.well-known/ { + error_page 404 = @wellknown-upstream; + proxy_pass http://127.0.0.1:3000/well-known-handler/; } <% if (proxyAuth.enabled) { %> diff --git a/src/routes/apps.js b/src/routes/apps.js index 35b6f0eef..e798d7bb9 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -32,6 +32,7 @@ exports = module.exports = { setLocation, setDataDir, setMounts, + setWellKnown, stop, start, @@ -44,6 +45,7 @@ exports = module.exports = { uploadFile, downloadFile, + load }; @@ -795,3 +797,19 @@ function setMounts(req, res, next) { next(new HttpSuccess(202, { taskId: result.taskId })); }); } + +function setWellKnown(req, res, next) { + assert.strictEqual(typeof req.body, 'object'); + assert.strictEqual(typeof req.resource, 'object'); + + if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object')); + if (req.body.wellKnown) { + if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings')); + } + + apps.setWellKnown(req.resource, req.body.wellKnown, auditSource.fromRequest(req), function (error, result) { + if (error) return next(BoxError.toHttpError(error)); + + next(new HttpSuccess(200, {})); + }); +} diff --git a/src/routes/index.js b/src/routes/index.js index 8c32856e7..9dd0f44ce 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -25,5 +25,6 @@ exports = module.exports = { tasks: require('./tasks.js'), tokens: require('./tokens.js'), users: require('./users.js'), - volumes: require('./volumes.js') + volumes: require('./volumes.js'), + wellknown: require('./wellknown.js') }; diff --git a/src/routes/wellknown.js b/src/routes/wellknown.js new file mode 100644 index 000000000..c17af49bd --- /dev/null +++ b/src/routes/wellknown.js @@ -0,0 +1,21 @@ +'use strict'; + +exports = module.exports = { + get +}; + +const apps = require('../apps.js'), + HttpError = require('connect-lastmile').HttpError; + +function get(req, res, next) { + const host = req.headers['host']; + + apps.getByFqdn(host, function (error, app) { + if (error) return next(new HttpError(404, error.message)); + + const location = req.params[0]; + if (!app.wellKnown || !(location in app.wellKnown)) return next(new HttpError(404, 'No custom well-known config')); + + res.status(200).send(app.wellKnown[location]); + }); +} diff --git a/src/server.js b/src/server.js index 6b537741b..dc9f98682 100644 --- a/src/server.js +++ b/src/server.js @@ -217,6 +217,7 @@ function initializeExpressSync() { router.post('/api/v1/apps/:id/configure/data_dir', json, token, authorizeAdmin, routes.apps.load, routes.apps.setDataDir); router.post('/api/v1/apps/:id/configure/location', json, token, authorizeAdmin, routes.apps.load, routes.apps.setLocation); router.post('/api/v1/apps/:id/configure/mounts', json, token, authorizeAdmin, routes.apps.load, routes.apps.setMounts); + router.post('/api/v1/apps/:id/well_known', json, token, authorizeAdmin, routes.apps.load, routes.apps.setWellKnown); router.post('/api/v1/apps/:id/repair', json, token, authorizeAdmin, routes.apps.load, routes.apps.repair); router.post('/api/v1/apps/:id/update', json, token, authorizeAdmin, routes.apps.load, routes.apps.update); router.post('/api/v1/apps/:id/restore', json, token, authorizeAdmin, routes.apps.load, routes.apps.restore); @@ -234,6 +235,7 @@ function initializeExpressSync() { router.post('/api/v1/apps/:id/upload', json, token, authorizeAdmin, multipart, routes.apps.load, routes.apps.uploadFile); router.use ('/api/v1/apps/:id/files/*', token, authorizeAdmin, routes.filemanager.proxy); router.get ('/api/v1/apps/:id/exec', token, authorizeAdmin, routes.apps.load, routes.apps.exec); + // websocket cannot do bearer authentication router.get ('/api/v1/apps/:id/execws', routes.accesscontrol.websocketAuth.bind(null, users.ROLE_ADMIN), routes.apps.load, routes.apps.execWebSocket); @@ -318,6 +320,9 @@ function initializeExpressSync() { router.get ('/api/v1/services/:service/logstream', token, authorizeAdmin, routes.services.getLogStream); router.post('/api/v1/services/:service/restart', json, token, authorizeAdmin, routes.services.restart); + // well known + router.get ('/well-known-handler/*', routes.wellknown.get); + // disable server socket "idle" timeout. we use the timeout middleware to handle timeouts on a route level // we rely on nginx for timeouts on the TCP level (see client_header_timeout) httpServer.setTimeout(0);