diff --git a/migrations/20201223014453-apps-add-wellKnownJson.js b/migrations/20201223014453-domains-add-wellKnownJson.js similarity index 72% rename from migrations/20201223014453-apps-add-wellKnownJson.js rename to migrations/20201223014453-domains-add-wellKnownJson.js index 300330ea8..6591b92eb 100644 --- a/migrations/20201223014453-apps-add-wellKnownJson.js +++ b/migrations/20201223014453-domains-add-wellKnownJson.js @@ -4,7 +4,7 @@ const async = require('async'), safe = require('safetydance'); exports.up = function(db, callback) { - db.runSql('ALTER TABLE apps ADD COLUMN wellKnownJson TEXT', function (error) { + db.runSql('ALTER TABLE domains ADD COLUMN wellKnownJson TEXT', function (error) { if (error) return callback(error); // keep the paths around, so that we don't need to trigger a re-configure. the old nginx config will use the paths @@ -27,10 +27,9 @@ exports.up = function(db, callback) { } async.eachSeries(Object.keys(wellKnown), function (fqdn, iteratorDone) { - db.runSql('SELECT appId FROM subdomains WHERE domain=? AND subdomain=?', [ fqdn, '' ], function (error, result) { - if (error || result.length === 0) return iteratorDone(); // there could be no corresponding app - - db.runSql('UPDATE apps SET wellKnownJson=? WHERE id=?', [ JSON.stringify(wellKnown), result[0].appId ], iteratorDone); + db.runSql('UPDATE domains SET wellKnownJson=? WHERE domain=?', [ JSON.stringify(wellKnown), fqdn ], function (error) { + if (error) console.error(error); // maybe the domain does not exist anymore + iteratorDone(); }); }, function (error) { callback(error); @@ -39,7 +38,7 @@ exports.up = function(db, callback) { }; exports.down = function(db, callback) { - db.runSql('ALTER TABLE apps DROP COLUMN wellknownJson', function (error) { + db.runSql('ALTER TABLE domains DROP COLUMN wellKnownJson', function (error) { if (error) console.error(error); callback(error); }); diff --git a/migrations/schema.sql b/migrations/schema.sql index d9c611378..b13aab5ff 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -86,7 +86,6 @@ CREATE TABLE IF NOT EXISTS apps( taskId INTEGER, // current task errorJson TEXT, servicesConfigJson TEXT, // app services configuration - wellKnownJson TEXT, containerIp VARCHAR(16) UNIQUE, // this is not-null because of ip allocation fails, user can 'repair' FOREIGN KEY(mailboxDomain) REFERENCES domains(domain), @@ -149,6 +148,7 @@ CREATE TABLE IF NOT EXISTS domains( provider VARCHAR(16) NOT NULL, configJson TEXT, /* JSON containing the dns backend provider config */ tlsConfigJson TEXT, /* JSON containing the tls provider config */ + wellKnownJson TEXT, /* JSON containing well known docs for this domain */ PRIMARY KEY (domain)) diff --git a/runTests b/runTests index beb7cda62..be76fcaa4 100755 --- a/runTests +++ b/runTests @@ -24,6 +24,7 @@ cd ${DATA_DIR} mkdir -p appsdata mkdir -p boxdata/profileicons boxdata/appicons boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/sftp/ssh mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks +mkdir -p /mnt/music /media/music # volume test # translations mkdir -p box/dashboard/dist/translation diff --git a/src/appdb.js b/src/appdb.js index 7fa135d70..6352491f4 100644 --- a/src/appdb.js +++ b/src/appdb.js @@ -41,7 +41,7 @@ var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationSta 'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares', 'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.sso', 'apps.debugModeJson', 'apps.enableBackup', 'apps.proxyAuth', 'apps.containerIp', - 'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate', 'apps.wellKnownJson', + 'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate', 'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(','); var PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(','); @@ -118,9 +118,6 @@ function postProcess(result) { result.mounts = volumeIds[0] === null ? [] : volumeIds.map((v, idx) => { return { volumeId: v, readOnly: !!volumeReadOnlys[idx] }; }); // NOTE: volumeIds is [null] when volumes of an app is empty - result.wellKnown = safe.JSON.parse(result.wellKnownJson); - delete result.wellKnownJson; - result.error = safe.JSON.parse(result.errorJson); delete result.errorJson; diff --git a/src/apps.js b/src/apps.js index 1ca43bdf2..ea03b0e02 100644 --- a/src/apps.js +++ b/src/apps.js @@ -20,7 +20,6 @@ exports = module.exports = { setMemoryLimit, setCpuShares, setMounts, - setWellKnown, setAutomaticBackup, setAutomaticUpdate, setReverseProxyConfig, @@ -406,7 +405,7 @@ function removeInternalFields(app) { 'location', 'domain', 'fqdn', 'mailboxName', 'mailboxDomain', 'accessRestriction', 'manifest', 'portBindings', 'iconUrl', 'memoryLimit', 'cpuShares', 'sso', 'debugMode', 'reverseProxyConfig', 'enableBackup', 'creationTime', 'updateTime', 'ts', 'tags', - 'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts', 'wellKnown'); + 'label', 'alternateDomains', 'env', 'enableAutomaticUpdate', 'dataDir', 'mounts'); } // non-admins can only see these @@ -983,22 +982,6 @@ function setMounts(app, mounts, auditSource, callback) { }); } -function setWellKnown(app, wellKnown, auditSource, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof wellKnown, 'object'); - assert.strictEqual(typeof auditSource, 'object'); - assert.strictEqual(typeof callback, 'function'); - - const appId = app.id; - appdb.update(appId, { wellKnown }, function (error) { - if (error) return callback(error); - - eventlog.add(eventlog.ACTION_APP_CONFIGURE, auditSource, { appId, app, wellKnown }); - - callback(); - }); -} - function setEnvironment(app, env, auditSource, callback) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof env, 'object'); diff --git a/src/domaindb.js b/src/domaindb.js index 578a081b7..cbeb1626f 100644 --- a/src/domaindb.js +++ b/src/domaindb.js @@ -16,14 +16,18 @@ var assert = require('assert'), database = require('./database.js'), safe = require('safetydance'); -var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson' ].join(','); +var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson' ].join(','); function postProcess(data) { data.config = safe.JSON.parse(data.configJson); - data.tlsConfig = safe.JSON.parse(data.tlsConfigJson); delete data.configJson; + + data.tlsConfig = safe.JSON.parse(data.tlsConfigJson); delete data.tlsConfigJson; + data.wellKnown = safe.JSON.parse(data.wellKnownJson); + delete data.wellKnownJson; + return data; } @@ -86,6 +90,9 @@ function update(name, domain, callback) { } else if (k === 'tlsConfig') { fields.push('tlsConfigJson = ?'); args.push(JSON.stringify(domain[k])); + } else if (k === 'wellKnown') { + fields.push('wellKnownJson = ?'); + args.push(JSON.stringify(domain[k])); } else { fields.push(k + ' = ?'); args.push(domain[k]); diff --git a/src/domains.js b/src/domains.js index 010e0a6fa..9ee7c61ae 100644 --- a/src/domains.js +++ b/src/domains.js @@ -1,32 +1,32 @@ 'use strict'; module.exports = exports = { - add: add, - get: get, - getAll: getAll, - update: update, - del: del, - clear: clear, + add, + get, + getAll, + update, + del, + clear, - fqdn: fqdn, - getName: getName, + fqdn, + getName, - getDnsRecords: getDnsRecords, - upsertDnsRecords: upsertDnsRecords, - removeDnsRecords: removeDnsRecords, + getDnsRecords, + upsertDnsRecords, + removeDnsRecords, - waitForDnsRecord: waitForDnsRecord, + waitForDnsRecord, - removePrivateFields: removePrivateFields, - removeRestrictedFields: removeRestrictedFields, + removePrivateFields, + removeRestrictedFields, - validateHostname: validateHostname, + validateHostname, - makeWildcard: makeWildcard, + makeWildcard, - parentDomain: parentDomain, + parentDomain, - checkDnsRecords: checkDnsRecords + checkDnsRecords }; var assert = require('assert'), @@ -152,6 +152,12 @@ function validateTlsConfig(tlsConfig, dnsProvider) { return null; } +function validateWellKnown(wellKnown) { + assert.strictEqual(typeof wellKnown, 'object'); + + return null; +} + function add(domain, data, auditSource, callback) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof data.zoneName, 'string'); @@ -246,7 +252,7 @@ function update(domain, data, auditSource, callback) { assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof callback, 'function'); - let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data; + let { zoneName, provider, config, fallbackCertificate, tlsConfig, wellKnown } = data; if (settings.isDemo() && (domain === settings.adminDomain())) return callback(new BoxError(BoxError.CONFLICT, 'Not allowed in demo mode')); @@ -267,6 +273,9 @@ function update(domain, data, auditSource, callback) { error = validateTlsConfig(tlsConfig, provider); if (error) return callback(error); + error = validateWellKnown(wellKnown, provider); + if (error) return callback(error); + if (provider === domainObject.provider) api(provider).injectPrivateFields(config, domainObject.config); verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) { @@ -274,9 +283,10 @@ function update(domain, data, auditSource, callback) { let newData = { config: sanitizedConfig, - zoneName: zoneName, - provider: provider, - tlsConfig: tlsConfig + zoneName, + provider, + tlsConfig, + wellKnown }; domaindb.update(domain, newData, function (error) { @@ -431,7 +441,7 @@ function waitForDnsRecord(location, domain, type, value, options, callback) { // removes all fields that are strictly private and should never be returned by API calls function removePrivateFields(domain) { - var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate'); + var result = _.pick(domain, 'domain', 'zoneName', 'provider', 'config', 'tlsConfig', 'fallbackCertificate', 'wellKnown'); return api(result.provider).removePrivateFields(result); } diff --git a/src/routes/apps.js b/src/routes/apps.js index e798d7bb9..a07bd318d 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -32,7 +32,6 @@ exports = module.exports = { setLocation, setDataDir, setMounts, - setWellKnown, stop, start, @@ -797,19 +796,3 @@ function setMounts(req, res, next) { next(new HttpSuccess(202, { taskId: result.taskId })); }); } - -function setWellKnown(req, res, next) { - assert.strictEqual(typeof req.body, 'object'); - assert.strictEqual(typeof req.resource, 'object'); - - if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object')); - if (req.body.wellKnown) { - if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings')); - } - - apps.setWellKnown(req.resource, req.body.wellKnown, auditSource.fromRequest(req), function (error, result) { - if (error) return next(BoxError.toHttpError(error)); - - next(new HttpSuccess(200, {})); - }); -} diff --git a/src/routes/domains.js b/src/routes/domains.js index 952ad8532..9c8fc2e2e 100644 --- a/src/routes/domains.js +++ b/src/routes/domains.js @@ -99,6 +99,13 @@ function update(req, res, next) { if (!req.body.tlsConfig.provider || typeof req.body.tlsConfig.provider !== 'string') return next(new HttpError(400, 'tlsConfig.provider must be a string')); } + if ('wellKnown' in req.body) { + if (typeof req.body.wellKnown !== 'object') return next(new HttpError(400, 'wellKnown must be an object')); + if (req.body.wellKnown) { + if (Object.keys(req.body.wellKnown).some(k => typeof req.body.wellKnown[k] !== 'string')) return next(new HttpError(400, 'wellKnown is a map of strings')); + } + } + // some DNS providers like DigitalOcean take a really long time to verify credentials (https://github.com/expressjs/timeout/issues/26) req.clearTimeout(); @@ -107,7 +114,8 @@ function update(req, res, next) { provider: req.body.provider, config: req.body.config, fallbackCertificate: req.body.fallbackCertificate || null, - tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' } + tlsConfig: req.body.tlsConfig || { provider: 'letsencrypt-prod' }, + wellKnown: req.body.wellKnown || null }; domains.update(req.params.domain, data, auditSource.fromRequest(req), function (error) { diff --git a/src/routes/wellknown.js b/src/routes/wellknown.js index c17af49bd..40cba64c2 100644 --- a/src/routes/wellknown.js +++ b/src/routes/wellknown.js @@ -4,18 +4,18 @@ exports = module.exports = { get }; -const apps = require('../apps.js'), +const domains = require('../domains.js'), HttpError = require('connect-lastmile').HttpError; function get(req, res, next) { const host = req.headers['host']; - apps.getByFqdn(host, function (error, app) { + domains.get(host, function (error, domain) { if (error) return next(new HttpError(404, error.message)); const location = req.params[0]; - if (!app.wellKnown || !(location in app.wellKnown)) return next(new HttpError(404, 'No custom well-known config')); + if (!domain.wellKnown || !(location in domain.wellKnown)) return next(new HttpError(404, 'No custom well-known config')); - res.status(200).send(app.wellKnown[location]); + res.status(200).send(domain.wellKnown[location]); }); } diff --git a/src/server.js b/src/server.js index dc9f98682..f2f297082 100644 --- a/src/server.js +++ b/src/server.js @@ -217,7 +217,6 @@ function initializeExpressSync() { router.post('/api/v1/apps/:id/configure/data_dir', json, token, authorizeAdmin, routes.apps.load, routes.apps.setDataDir); router.post('/api/v1/apps/:id/configure/location', json, token, authorizeAdmin, routes.apps.load, routes.apps.setLocation); router.post('/api/v1/apps/:id/configure/mounts', json, token, authorizeAdmin, routes.apps.load, routes.apps.setMounts); - router.post('/api/v1/apps/:id/well_known', json, token, authorizeAdmin, routes.apps.load, routes.apps.setWellKnown); router.post('/api/v1/apps/:id/repair', json, token, authorizeAdmin, routes.apps.load, routes.apps.repair); router.post('/api/v1/apps/:id/update', json, token, authorizeAdmin, routes.apps.load, routes.apps.update); router.post('/api/v1/apps/:id/restore', json, token, authorizeAdmin, routes.apps.load, routes.apps.restore); @@ -298,12 +297,12 @@ function initializeExpressSync() { router.post('/api/v1/support/remote_support', json, token, authorizeAdmin, routes.support.canEnableRemoteSupport, routes.support.enableRemoteSupport); // domain routes - router.post('/api/v1/domains', json, token, authorizeAdmin, routes.domains.add); - router.get ('/api/v1/domains', token, routes.domains.getAll); - router.get ('/api/v1/domains/:domain', token, authorizeAdmin, routes.domains.get); // this is manage scope because it returns non-restricted fields - router.put ('/api/v1/domains/:domain', json, token, authorizeAdmin, routes.domains.update); - router.del ('/api/v1/domains/:domain', token, authorizeAdmin, routes.domains.del); - router.get ('/api/v1/domains/:domain/dns_check', token, authorizeAdmin, routes.domains.checkDnsRecords); + router.post('/api/v1/domains', json, token, authorizeAdmin, routes.domains.add); + router.get ('/api/v1/domains', token, routes.domains.getAll); + router.get ('/api/v1/domains/:domain', token, authorizeAdmin, routes.domains.get); // this is manage scope because it returns non-restricted fields + router.put ('/api/v1/domains/:domain', json, token, authorizeAdmin, routes.domains.update); + router.del ('/api/v1/domains/:domain', token, authorizeAdmin, routes.domains.del); + router.get ('/api/v1/domains/:domain/dns_check', token, authorizeAdmin, routes.domains.checkDnsRecords); // volume routes router.post('/api/v1/volumes', json, token, authorizeAdmin, routes.volumes.add); diff --git a/src/test/apps-test.js b/src/test/apps-test.js index 815603760..7459f10f2 100644 --- a/src/test/apps-test.js +++ b/src/test/apps-test.js @@ -83,7 +83,8 @@ describe('Apps', function () { provider: 'noop', config: { }, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; const DOMAIN_1 = { @@ -92,7 +93,8 @@ describe('Apps', function () { provider: 'noop', config: { }, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; var APP_0 = { diff --git a/src/test/apptask-test.js b/src/test/apptask-test.js index bd3a62a80..f64a6ee30 100644 --- a/src/test/apptask-test.js +++ b/src/test/apptask-test.js @@ -58,7 +58,8 @@ const DOMAIN_0 = { endpoint: 'http://localhost:5353' }, fallbackCertificate: null, - tlsConfig: { provider: 'letsencrypt-staging' } + tlsConfig: { provider: 'letsencrypt-staging' }, + wellKnown: null }; let AUDIT_SOURCE = { ip: '1.2.3.4' }; diff --git a/src/test/backups-test.js b/src/test/backups-test.js index b78188b10..fd901973b 100644 --- a/src/test/backups-test.js +++ b/src/test/backups-test.js @@ -168,7 +168,8 @@ describe('backups', function () { provider: 'manual', config: { }, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; const AUDIT_SOURCE = { ip: '1.2.3.4' }; diff --git a/src/test/database-test.js b/src/test/database-test.js index b76f727f3..875360276 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -85,7 +85,8 @@ const DOMAIN_0 = { zoneName: 'foobar.com', provider: 'digitalocean', config: { token: 'abcd' }, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; const DOMAIN_1 = { @@ -93,7 +94,8 @@ const DOMAIN_1 = { zoneName: 'cloudron.io', provider: 'manual', config: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; describe('database', function () { diff --git a/src/test/dns-test.js b/src/test/dns-test.js index bb32e82a4..460eb23a6 100644 --- a/src/test/dns-test.js +++ b/src/test/dns-test.js @@ -22,7 +22,8 @@ var DOMAIN_0 = { provider: 'noop', config: {}, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; var AUDIT_SOURCE = { ip: '1.2.3.4' }; diff --git a/src/test/externalldap-test.js b/src/test/externalldap-test.js index d8837b0d3..124af095e 100644 --- a/src/test/externalldap-test.js +++ b/src/test/externalldap-test.js @@ -40,7 +40,8 @@ const DOMAIN_0 = { provider: 'manual', config: {}, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; const LDAP_SHARED_PASSWORD = 'validpassword'; diff --git a/src/test/groups-test.js b/src/test/groups-test.js index 352ead4b1..490c66c7a 100644 --- a/src/test/groups-test.js +++ b/src/test/groups-test.js @@ -24,7 +24,8 @@ var GROUP1_NAME = 'externs', const DOMAIN_0 = { domain: 'example.com', zoneName: 'example.com', - config: { provider: 'manual' } + config: { provider: 'manual' }, + wellKnown: null }; var USER_0 = { diff --git a/src/test/ldap-test.js b/src/test/ldap-test.js index b7d041745..c75d1d2b5 100644 --- a/src/test/ldap-test.js +++ b/src/test/ldap-test.js @@ -29,7 +29,8 @@ const DOMAIN_0 = { config: {}, provider: 'manual', fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; // owner diff --git a/src/test/mail-test.js b/src/test/mail-test.js index 0654e4b87..9cd224756 100644 --- a/src/test/mail-test.js +++ b/src/test/mail-test.js @@ -21,7 +21,8 @@ const DOMAIN_0 = { provider: 'manual', config: {}, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; const AUDIT_SOURCE = { diff --git a/src/test/reverseproxy-test.js b/src/test/reverseproxy-test.js index 2a3fc8468..39d65f433 100644 --- a/src/test/reverseproxy-test.js +++ b/src/test/reverseproxy-test.js @@ -18,7 +18,8 @@ const DOMAIN_0 = { provider: 'noop', config: {}, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; let AUDIT_SOURCE = { ip: '1.2.3.4' }; diff --git a/src/test/updatechecker-test.js b/src/test/updatechecker-test.js index 325128b09..49616c83b 100644 --- a/src/test/updatechecker-test.js +++ b/src/test/updatechecker-test.js @@ -41,7 +41,8 @@ const DOMAIN_0 = { config: {}, provider: 'manual', fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; var AUDIT_SOURCE = { diff --git a/src/test/users-test.js b/src/test/users-test.js index 3b38d7459..f6df12bbe 100644 --- a/src/test/users-test.js +++ b/src/test/users-test.js @@ -37,7 +37,8 @@ const DOMAIN_0 = { provider: 'manual', config: {}, fallbackCertificate: null, - tlsConfig: { provider: 'fallback' } + tlsConfig: { provider: 'fallback' }, + wellKnown: null }; function cleanupUsers(done) { diff --git a/src/test/volumes-test.js b/src/test/volumes-test.js index 2395aa674..72f1250a8 100644 --- a/src/test/volumes-test.js +++ b/src/test/volumes-test.js @@ -66,6 +66,7 @@ describe('Volumes', function () { it('cannot add duplicate name', function (done) { volumes.add('music', '/media/music', AUDIT_SOURCE, function (error) { + console.dir(error); expect(error.reason).to.be(BoxError.ALREADY_EXISTS); done(); }); @@ -110,4 +111,4 @@ describe('Volumes', function () { done(); }); }); -}); \ No newline at end of file +});