diff --git a/migrations/20210504221634-settings-migrate-firewall.js b/migrations/20210504221634-settings-migrate-firewall.js index 9bcb2e938..864be5e83 100644 --- a/migrations/20210504221634-settings-migrate-firewall.js +++ b/migrations/20210504221634-settings-migrate-firewall.js @@ -8,6 +8,8 @@ const BOX_DATA_DIR = '/home/yellowtent/boxdata'; const PLATFORM_DATA_DIR = '/home/yellowtent/platformdata'; exports.up = function (db, callback) { + if (!fs.existsSync(`${BOX_DATA_DIR}/firewall`)) return callback(); + const ports = safe.fs.readFileSync(`${BOX_DATA_DIR}/firewall/ports.json`); if (ports) { safe.fs.writeFileSync(`${PLATFORM_DATA_DIR}/firewall/ports.json`, ports); diff --git a/migrations/20210504230054-domains-add-fallbackCertificateJson.js b/migrations/20210504230054-domains-add-fallbackCertificateJson.js new file mode 100644 index 000000000..5627ca149 --- /dev/null +++ b/migrations/20210504230054-domains-add-fallbackCertificateJson.js @@ -0,0 +1,30 @@ +'use strict'; + +const async = require('async'), + fs = require('fs'); + +const CERTS_DIR = '/home/yellowtent/boxdata/certs'; + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE domains ADD COLUMN fallbackCertificateJson TEXT', function (error) { + if (error) return callback(error); + + db.all('SELECT * FROM domains', [ ], function (error, domains) { + if (error) return callback(error); + + async.eachSeries(domains, function (domain, iteratorDone) { + const cert = fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.cert`, 'utf8'); + const key = fs.readFileSync(`${CERTS_DIR}/${domain.domain}.host.key`, 'utf8'); + const fallbackCertificate = { cert, key }; + + db.runSql('UPDATE domains SET fallbackCertificateJson=? WHERE domain=?', [ JSON.stringify(fallbackCertificate), domain.domain ], iteratorDone); + }, callback); + }); + }); +}; + +exports.down = function(db, callback) { + async.series([ + db.runSql.run(db, 'ALTER TABLE domains DROP COLUMN fallbackCertificateJson') + ], callback); +}; diff --git a/migrations/schema.sql b/migrations/schema.sql index 3d18c5361..637f67467 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -157,6 +157,8 @@ CREATE TABLE IF NOT EXISTS domains( tlsConfigJson TEXT, /* JSON containing the tls provider config */ wellKnownJson TEXT, /* JSON containing well known docs for this domain */ + fallbackCertificateJson TEXT, + PRIMARY KEY (domain)) /* the default db collation is utf8mb4_unicode_ci but for the app table domain constraint we have to use the old one */ diff --git a/runTests b/runTests index 8b86a1675..8f12d2556 100755 --- a/runTests +++ b/runTests @@ -22,8 +22,8 @@ fi mkdir -p ${DATA_DIR} cd ${DATA_DIR} mkdir -p appsdata -mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com boxdata/firewall -mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh +mkdir -p boxdata/mail boxdata/certs boxdata/mail/dkim/localhost boxdata/mail/dkim/foobar.com +mkdir -p platformdata/addons/mail/banner platformdata/nginx/cert platformdata/nginx/applications platformdata/collectd/collectd.conf.d platformdata/addons platformdata/logrotate.d platformdata/backup platformdata/logs/tasks platformdata/sftp/ssh platformdata/firewall sudo mkdir -p /mnt/cloudron-test-music /media/cloudron-test-music # volume test # translations diff --git a/setup/start.sh b/setup/start.sh index e1a865de8..9b09e9969 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -63,6 +63,7 @@ mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \ mkdir -p "${PLATFORM_DATA_DIR}/update" mkdir -p "${PLATFORM_DATA_DIR}/sftp/ssh" # sftp keys mkdir -p "${PLATFORM_DATA_DIR}/firewall" +mkdir -p "${PLATFORM_DATA_DIR}/certs" mkdir -p "${BOX_DATA_DIR}/certs" mkdir -p "${BOX_DATA_DIR}/mail/dkim" diff --git a/src/domaindb.js b/src/domaindb.js index f854a4ab2..c003d1f7c 100644 --- a/src/domaindb.js +++ b/src/domaindb.js @@ -11,12 +11,12 @@ exports = module.exports = { clear }; -var assert = require('assert'), +const assert = require('assert'), BoxError = require('./boxerror.js'), database = require('./database.js'), safe = require('safetydance'); -var DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson' ].join(','); +const DOMAINS_FIELDS = [ 'domain', 'zoneName', 'provider', 'configJson', 'tlsConfigJson', 'wellKnownJson', 'fallbackCertificateJson' ].join(','); function postProcess(data) { data.config = safe.JSON.parse(data.configJson); @@ -28,6 +28,9 @@ function postProcess(data) { data.wellKnown = safe.JSON.parse(data.wellKnownJson); delete data.wellKnownJson; + data.fallbackCertificate = safe.JSON.parse(data.fallbackCertificateJson); + delete data.fallbackCertificateJson; + return data; } @@ -62,10 +65,12 @@ function add(name, data, callback) { assert.strictEqual(typeof data.provider, 'string'); assert.strictEqual(typeof data.config, 'object'); assert.strictEqual(typeof data.tlsConfig, 'object'); + assert.strictEqual(typeof data.fallbackCertificate, 'object'); assert.strictEqual(typeof callback, 'function'); let queries = [ - { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig) ] }, + { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson, fallbackCertificateJson) VALUES (?, ?, ?, ?, ?, ?)', + args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig), JSON.stringify(data.fallbackCertificate) ] }, { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] }, ]; @@ -84,14 +89,8 @@ function update(name, domain, callback) { var args = [ ], fields = [ ]; for (var k in domain) { - if (k === 'config') { - fields.push('configJson = ?'); - args.push(JSON.stringify(domain[k])); - } else if (k === 'tlsConfig') { - fields.push('tlsConfigJson = ?'); - args.push(JSON.stringify(domain[k])); - } else if (k === 'wellKnown') { - fields.push('wellKnownJson = ?'); + if (k === 'config' || k === 'tlsConfig' || k === 'wellKnown' || k === 'fallbackCertificate') { // json fields + fields.push(`${k}Json = ?`); args.push(JSON.stringify(domain[k])); } else { fields.push(k + ' = ?'); diff --git a/src/domains.js b/src/domains.js index c702e27e3..494f7c3c0 100644 --- a/src/domains.js +++ b/src/domains.js @@ -44,7 +44,6 @@ const apps = require('./apps.js'), eventlog = require('./eventlog.js'), mail = require('./mail.js'), reverseProxy = require('./reverseproxy.js'), - safe = require('safetydance'), settings = require('./settings.js'), sysinfo = require('./sysinfo.js'), tld = require('tldjs'), @@ -190,7 +189,7 @@ function add(domain, data, auditSource, callback) { let error = reverseProxy.validateCertificate('test', { domain, config }, fallbackCertificate); if (error) return callback(error); } else { - fallbackCertificate = reverseProxy.generateFallbackCertificateSync({ domain, config }); + fallbackCertificate = reverseProxy.generateFallbackCertificateSync(domain); if (fallbackCertificate.error) return callback(fallbackCertificate.error); } @@ -206,7 +205,7 @@ function add(domain, data, auditSource, callback) { verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) { if (error) return callback(error); - domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector }, function (error) { + domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector, fallbackCertificate }, function (error) { if (error) return callback(error); reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) { @@ -229,17 +228,7 @@ function get(domain, callback) { domaindb.get(domain, function (error, result) { if (error) return callback(error); - reverseProxy.getFallbackCertificate(domain, function (_, bundle) { // never returns an error - var cert = safe.fs.readFileSync(bundle.certFilePath, 'utf-8'); - var key = safe.fs.readFileSync(bundle.keyFilePath, 'utf-8'); - - // do not error here. otherwise, there is no way to fix things up from the UI - if (!cert || !key) debug(`Unable to read fallback certificates of ${domain} from disk`); - - result.fallbackCertificate = { cert: cert, key: key }; - - return callback(null, result); - }); + return callback(null, result); }); } @@ -297,9 +286,11 @@ function update(domain, data, auditSource, callback) { zoneName, provider, tlsConfig, - wellKnown + wellKnown, }; + if (fallbackCertificate) newData.fallbackCertificate = fallbackCertificate; + domaindb.update(domain, newData, function (error) { if (error) return callback(error); diff --git a/src/provision.js b/src/provision.js index 1e52434b7..6ad4c9674 100644 --- a/src/provision.js +++ b/src/provision.js @@ -7,7 +7,7 @@ exports = module.exports = { getStatus }; -var assert = require('assert'), +const assert = require('assert'), async = require('async'), backups = require('./backups.js'), BoxError = require('./boxerror.js'), @@ -18,6 +18,7 @@ var assert = require('assert'), domains = require('./domains.js'), eventlog = require('./eventlog.js'), mail = require('./mail.js'), + reverseProxy = require('./reverseproxy.js'), semver = require('semver'), settings = require('./settings.js'), sysinfo = require('./sysinfo.js'), @@ -201,6 +202,7 @@ function restore(backupConfig, backupId, version, sysinfoConfig, options, auditS setProgress.bind(null, 'restore', 'Downloading backup'), backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)), settings.setSysinfoConfig.bind(null, sysinfoConfig), + reverseProxy.restoreFallbackCertificates, (done) => { const adminDomain = settings.adminDomain(); // load this fresh from after the backup.restore async.series([ diff --git a/src/reverseproxy.js b/src/reverseproxy.js index 04fa9cd35..8fd8f45e8 100644 --- a/src/reverseproxy.js +++ b/src/reverseproxy.js @@ -2,7 +2,6 @@ exports = module.exports = { setFallbackCertificate, - getFallbackCertificate, generateFallbackCertificateSync, setAppCertificateSync, @@ -25,6 +24,7 @@ exports = module.exports = { writeAppConfig, removeAppConfigs, + restoreFallbackCertificates, // exported for testing _getAcmeApi: getAcmeApi @@ -196,10 +196,9 @@ function reload(callback) { }); } -function generateFallbackCertificateSync(domainObject) { - assert.strictEqual(typeof domainObject, 'object'); +function generateFallbackCertificateSync(domain) { + assert.strictEqual(typeof domain, 'string'); - const domain = domainObject.domain; const certFilePath = path.join(os.tmpdir(), `${domain}-${crypto.randomBytes(4).readUInt32LE(0)}.cert`); const keyFilePath = path.join(os.tmpdir(), `${domain}-${crypto.randomBytes(4).readUInt32LE(0)}.key`); @@ -208,7 +207,7 @@ function generateFallbackCertificateSync(domainObject) { let opensslConfWithSan; let cn = domain; - debug(`generateFallbackCertificateSync: domain=${domainObject.domain} cn=${cn}`); + debug(`generateFallbackCertificateSync: domain=${domain} cn=${cn}`); opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`; let configFile = path.join(os.tmpdir(), 'openssl-' + crypto.randomBytes(4).readUInt32LE(0) + '.conf'); @@ -247,14 +246,28 @@ function setFallbackCertificate(domain, fallback, callback) { }); } -function getFallbackCertificate(domain, callback) { - assert.strictEqual(typeof domain, 'string'); +function restoreFallbackCertificates(callback) { assert.strictEqual(typeof callback, 'function'); + domains.getAll(function (error, result) { + if (error) return callback(error); + + result.forEach(function (domain) { + if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${domain.domain}.host.cert`), domain.fallbackCertificate.cert)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message)); + if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${domain.domain}.host.key`), domains.fallbackCertificate.key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message)); + }); + + callback(null); + }); +} + +function getFallbackCertificatePathSync(domain) { + assert.strictEqual(typeof domain, 'string'); + const certFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.cert`); const keyFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.key`); - callback(null, { certFilePath, keyFilePath }); + return { certFilePath, keyFilePath }; } function setAppCertificateSync(location, domainObject, certificate) { @@ -315,12 +328,12 @@ function getCertificate(fqdn, domain, callback) { if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath }); - if (domainObject.tlsConfig.provider === 'fallback') return getFallbackCertificate(domain, callback); + if (domainObject.tlsConfig.provider === 'fallback') return callback(null, getFallbackCertificatePathSync(domain)); getAcmeCertificate(fqdn, domainObject, function (error, result) { if (error || result) return callback(error, result); - return getFallbackCertificate(domain, callback); + return callback(null, getFallbackCertificatePathSync(domain)); }); }); } @@ -346,11 +359,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) { if (domainObject.tlsConfig.provider === 'fallback') { debug(`ensureCertificate: ${vhost} will use fallback certs`); - return getFallbackCertificate(domain, function (error, bundle) { - if (error) return callback(error); - - callback(null, bundle, { renewed: false }); - }); + return callback(null, getFallbackCertificatePathSync(domain), { renewed: false }); } getAcmeApi(domainObject, function (error, acmeApi, apiOptions) { @@ -382,11 +391,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) { debug(`ensureCertificate: renewal of ${vhost} failed. using fallback certificates for ${domain}`); - getFallbackCertificate(domain, function (error, bundle) { - if (error) return callback(error); - - callback(null, bundle, { renewed: false }); - }); + callback(null, getFallbackCertificatePathSync(domain), { renewed: false }); }); }); }); diff --git a/src/routes/domains.js b/src/routes/domains.js index d9a9665d0..7f669b049 100644 --- a/src/routes/domains.js +++ b/src/routes/domains.js @@ -10,7 +10,7 @@ exports = module.exports = { checkDnsRecords, }; -var assert = require('assert'), +const assert = require('assert'), auditSource = require('../auditsource.js'), BoxError = require('../boxerror.js'), domains = require('../domains.js'), diff --git a/src/test/apps-test.js b/src/test/apps-test.js index f524f648b..0b949efdb 100644 --- a/src/test/apps-test.js +++ b/src/test/apps-test.js @@ -5,9 +5,10 @@ 'use strict'; -var appdb = require('../appdb.js'), +const appdb = require('../appdb.js'), apps = require('../apps.js'), async = require('async'), + blobs = require('../blobs.js'), BoxError = require('../boxerror.js'), database = require('../database.js'), domains = require('../domains.js'), @@ -175,6 +176,7 @@ describe('Apps', function () { async.series([ database.initialize, database._clear, + blobs.initSecrets, provision.setup.bind(null, DOMAIN_0, { provider: 'generic' }, AUDIT_SOURCE), domains.add.bind(null, DOMAIN_1.domain, DOMAIN_1, AUDIT_SOURCE), userdb.add.bind(null, ADMIN_0.id, ADMIN_0), diff --git a/src/test/database-test.js b/src/test/database-test.js index cd92315e5..f0db826f3 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -5,7 +5,7 @@ 'use strict'; -var appdb = require('../appdb.js'), +const appdb = require('../appdb.js'), apps = require('../apps.js'), async = require('async'), backupdb = require('../backupdb.js'), @@ -21,6 +21,7 @@ var appdb = require('../appdb.js'), mailboxdb = require('../mailboxdb.js'), maildb = require('../maildb.js'), notificationdb = require('../notificationdb.js'), + reverseProxy = require('../reverseproxy.js'), settingsdb = require('../settingsdb.js'), taskdb = require('../taskdb.js'), tokendb = require('../tokendb.js'), @@ -92,6 +93,7 @@ const DOMAIN_0 = { tlsConfig: { provider: 'fallback' }, wellKnown: null }; +DOMAIN_0.fallbackCertificate = reverseProxy.generateFallbackCertificateSync(DOMAIN_0.domain); const DOMAIN_1 = { domain: 'foo.cloudron.io', @@ -101,6 +103,7 @@ const DOMAIN_1 = { tlsConfig: { provider: 'fallback' }, wellKnown: null }; +DOMAIN_1.fallbackCertificate = reverseProxy.generateFallbackCertificateSync(DOMAIN_1.domain); describe('database', function () { before(function (done) { @@ -311,15 +314,15 @@ describe('database', function () { }); it('can add domain', function (done) { - domaindb.add(DOMAIN_0.domain, { zoneName: DOMAIN_0.zoneName, provider: DOMAIN_0.provider, config: DOMAIN_0.config, tlsConfig: DOMAIN_0.tlsConfig }, done); + domaindb.add(DOMAIN_0.domain, DOMAIN_0, done); }); it('can add another domain', function (done) { - domaindb.add(DOMAIN_1.domain, { zoneName: DOMAIN_1.zoneName, provider: DOMAIN_1.provider, config: DOMAIN_1.config, tlsConfig: DOMAIN_1.tlsConfig }, done); + domaindb.add(DOMAIN_1.domain, DOMAIN_1, done); }); it('cannot add same domain twice', function (done) { - domaindb.add(DOMAIN_0.domain, { zoneName: DOMAIN_0.zoneName, provider: DOMAIN_0.provider, config: DOMAIN_0.config, tlsConfig: DOMAIN_0.tlsConfig }, function (error) { + domaindb.add(DOMAIN_0.domain, DOMAIN_0, function (error) { expect(error).to.be.ok(); expect(error.reason).to.be(BoxError.ALREADY_EXISTS); done(); @@ -952,7 +955,7 @@ describe('database', function () { before(function (done) { async.series([ userdb.add.bind(null, USER_0.id, USER_0), - domaindb.add.bind(null, DOMAIN_0.domain, { zoneName: DOMAIN_0.zoneName, provider: DOMAIN_0.provider, config: DOMAIN_0.config, tlsConfig: DOMAIN_0.tlsConfig }) + domaindb.add.bind(null, DOMAIN_0.domain, DOMAIN_0) ], done); }); @@ -1840,7 +1843,7 @@ describe('database', function () { describe('mailboxes', function () { before(function (done) { async.series([ - domaindb.add.bind(null, DOMAIN_0.domain, { zoneName: DOMAIN_0.zoneName, provider: DOMAIN_0.provider, config: DOMAIN_0.config, tlsConfig: DOMAIN_0.tlsConfig }), + domaindb.add.bind(null, DOMAIN_0.domain, DOMAIN_0), ], done); }); @@ -2013,7 +2016,7 @@ describe('database', function () { }; before(function (done) { - domaindb.add(DOMAIN_0.domain, { zoneName: DOMAIN_0.zoneName, provider: DOMAIN_0.provider, config: DOMAIN_0.config, tlsConfig: DOMAIN_0.tlsConfig }, done); + domaindb.add(DOMAIN_0.domain, DOMAIN_0, done); }); after(function (done) {