From a2a1d842fad8950ad47bd58c2bab341f23c8e0de Mon Sep 17 00:00:00 2001 From: Johannes Zellner Date: Mon, 30 Oct 2017 00:16:33 +0100 Subject: [PATCH] Add db migration scripts This adds domains table and adjusts the apps and mailboxes table accordingly Also ensure we explicitly set the table collation, this is required for the foreign key from apps table (utf8) and the newly created domains table, which by default now would be utf8mb4 Put db table constraint for mailboxes.domain Update the schema file --- migrations/20171026212925-apps-add-domain.js | 81 +++++++++++++++++++ .../20171027212823-domains-add-table.js | 59 +++++++++++--- ...171029214604-apps-add-domain-constraint.js | 15 ++++ ...3212918-mailboxes-add-domain-constraint.js | 15 ++++ migrations/schema.sql | 13 ++- src/platform.js | 25 ------ 6 files changed, 170 insertions(+), 38 deletions(-) create mode 100644 migrations/20171026212925-apps-add-domain.js create mode 100644 migrations/20171029214604-apps-add-domain-constraint.js create mode 100644 migrations/20171103212918-mailboxes-add-domain-constraint.js diff --git a/migrations/20171026212925-apps-add-domain.js b/migrations/20171026212925-apps-add-domain.js new file mode 100644 index 000000000..6bd1bc0b4 --- /dev/null +++ b/migrations/20171026212925-apps-add-domain.js @@ -0,0 +1,81 @@ +'use strict'; + +var async = require('async'), + safe = require('safetydance'); + +exports.up = function(db, callback) { + function prepareTestSetupIfNeeded(done) { + if (process.env.BOX_ENV !== 'test') return done(); + + const settings = [ + [ 'domain', JSON.stringify({ fqdn: 'example.com', zoneName: 'example.com' })], + [ 'dns_config', JSON.stringify({ provider: 'manual', wildcard: true })] + ]; + + async.eachSeries(settings, function (setting, callback) { + db.runSql('INSERT INTO settings (name, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value)', setting, callback); + }, done); + } + + prepareTestSetupIfNeeded(function (error) { + if (error) return callback(error); + + // first check precondtion of domain entry in settings + db.all('SELECT * FROM settings WHERE name = ?', [ 'domain' ], function (error, result) { + if (error) return callback(error); + if (result.length === 0 || !result[0].value) return callback(new Error('no domain entry in settings table')); + + var domain = safe.JSON.parse(result[0].value); + if (!domain) return callback(new Error('Unable to parse domain entry from settings table. Invalid JSON.')); + + // if no domain has been set we can't continue + if (!domain.fqdn) return callback(new Error('no fqdn value in domain settings entry')); + + async.series([ + db.runSql.bind(db, 'START TRANSACTION;'), + function addAppsDomainColumn(done) { + db.runSql('ALTER TABLE apps ADD COLUMN domain VARCHAR(128)', [], done); + }, + function setAppDomain(done) { + db.runSql('UPDATE apps SET domain = ?', [ domain.fqdn ], done); + }, + function addAppsLocationDomainUniqueConstraint(done) { + db.runSql('ALTER TABLE apps ADD UNIQUE location_domain_unique_index (location, domain)', [], done); + }, + function addMailboxesDomainColumn(done) { + db.runSql('ALTER TABLE mailboxes ADD COLUMN domain VARCHAR(128)', [], done); + }, + function setMailboxesDomain(done) { + db.runSql('UPDATE mailboxes SET domain = ?', [ domain.fqdn ], done); + }, + function dropAppsLocationUniqueConstraint(done) { + db.runSql('ALTER TABLE apps DROP INDEX location', [], done); + }, + db.runSql.bind(db, 'COMMIT') + ], callback); + }); + }); +}; + +exports.down = function(db, callback) { + async.series([ + db.runSql.bind(db, 'START TRANSACTION;'), + function dropMailboxesDomainColumn(done) { + // done(); + db.runSql('ALTER TABLE mailboxes DROP COLUMN domain', [], done); + }, + function dropLocationDomainUniqueConstraint(done) { + // done(); + db.runSql('ALTER TABLE apps DROP INDEX location_domain_unique_index', [], done); + }, + function dropAppsDomainColumn(done) { + // done(); + db.runSql('ALTER TABLE apps DROP COLUMN domain', [], done); + }, + function addAppsLocationUniqueConstraint(done) { + // done(); + db.runSql('ALTER TABLE apps ADD UNIQUE location (location)', [], done); + }, + db.runSql.bind(db, 'COMMIT') + ], callback); +}; diff --git a/migrations/20171027212823-domains-add-table.js b/migrations/20171027212823-domains-add-table.js index cca8d1c95..593182359 100644 --- a/migrations/20171027212823-domains-add-table.js +++ b/migrations/20171027212823-domains-add-table.js @@ -1,16 +1,55 @@ 'use strict'; -exports.up = function(db, callback) { - var cmd = "CREATE TABLE domains(" + - "domain VARCHAR(128) NOT NULL," + - "zoneName VARCHAR(128) NOT NULL," + - "configJson TEXT," + - "PRIMARY KEY (domain))"; +var async = require('async'), + safe = require('safetydance'); - db.runSql(cmd, function (error) { - if (error) console.error(error); - callback(error); - }); +exports.up = function(db, callback) { + var fqdn, zoneName, configJson; + + async.series([ + function gatherDomain(done) { + db.all('SELECT * FROM settings WHERE name = ?', [ 'domain' ], function (error, result) { + if (error) return done(error); + if (result.length === 0 || !result[0].value) return done(new Error('no domain entry in settings table')); + + var domain = safe.JSON.parse(result[0].value); + if (!domain) return done(new Error('Unable to parse domain entry from settings table. Invalid JSON.')); + + // if no domain has been set we can't continue + if (!domain.fqdn) return done(new Error('no fqdn value in domain settings entry')); + + fqdn = domain.fqdn; + zoneName = domain.zoneName || fqdn; + + done(); + }); + }, + function gatherDNSConfig(done) { + db.all('SELECT * FROM settings WHERE name = ?', [ 'dns_config' ], function (error, result) { + if (error ) return done(error); + + configJson = (result[0] && result[0].value) ? result[0].value : JSON.stringify({ provider: 'manual'}); + + done(); + }); + }, + db.runSql.bind(db, 'START TRANSACTION;'), + function createDomainsTable(done) { + var cmd = ` + CREATE TABLE domains( + domain VARCHAR(128) NOT NULL UNIQUE, + zoneName VARCHAR(128) NOT NULL, + configJson TEXT, + PRIMARY KEY (domain)) CHARACTER SET utf8 COLLATE utf8_bin + `; + + db.runSql(cmd, [], done); + }, + function addInitialDomain(done) { + db.runSql('INSERT INTO domains (domain, zoneName, configJson) VALUES (?, ?, ?)', [ fqdn, zoneName, configJson ], done); + }, + db.runSql.bind(db, 'COMMIT') + ], callback); }; exports.down = function(db, callback) { diff --git a/migrations/20171029214604-apps-add-domain-constraint.js b/migrations/20171029214604-apps-add-domain-constraint.js new file mode 100644 index 000000000..7743bf4d5 --- /dev/null +++ b/migrations/20171029214604-apps-add-domain-constraint.js @@ -0,0 +1,15 @@ +'use strict'; + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE apps ADD CONSTRAINT apps_domain_constraint FOREIGN KEY(domain) REFERENCES domains(domain)', function (error) { + if (error) console.error(error); + callback(error); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE apps DROP FOREIGN KEY apps_domain_constraint', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/migrations/20171103212918-mailboxes-add-domain-constraint.js b/migrations/20171103212918-mailboxes-add-domain-constraint.js new file mode 100644 index 000000000..4cca04c00 --- /dev/null +++ b/migrations/20171103212918-mailboxes-add-domain-constraint.js @@ -0,0 +1,15 @@ +'use strict'; + +exports.up = function(db, callback) { + db.runSql('ALTER TABLE mailboxes ADD CONSTRAINT mailboxes_domain_constraint FOREIGN KEY(domain) REFERENCES domains(domain)', function (error) { + if (error) console.error(error); + callback(error); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE mailboxes DROP FOREIGN KEY mailboxes_domain_constraint', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/migrations/schema.sql b/migrations/schema.sql index 213680abb..1ce6ce7f0 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -63,7 +63,8 @@ CREATE TABLE IF NOT EXISTS apps( containerId VARCHAR(128), manifestJson TEXT, httpPort INTEGER, // this is the nginx proxy port and not manifest.httpPort - location VARCHAR(128) NOT NULL UNIQUE, + location VARCHAR(128) NOT NULL, + domain VARCHAR(128) NOT NULL, dnsRecordId VARCHAR(512), // tracks any id that we got back to track dns updates accessRestrictionJson TEXT, // { users: [ ], groups: [ ] } createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -81,6 +82,7 @@ CREATE TABLE IF NOT EXISTS apps( oldConfigJson TEXT, // used to pass old config for apptask (configure, restore) updateConfigJson TEXT, // used to pass new config for apptask (update) + FOREIGN KEY(domain) REFERENCES domains(domain), PRIMARY KEY(id)); CREATE TABLE IF NOT EXISTS appPortBindings( @@ -140,12 +142,17 @@ CREATE TABLE IF NOT EXISTS mailboxes( ownerType VARCHAR(16) NOT NULL, /* 'app' or 'user' or 'group' */ aliasTarget VARCHAR(128), /* the target name type is an alias */ creationTime TIMESTAMP, + domain VARCHAR(128), + FOREIGN KEY(domain) REFERENCES domains(domain), PRIMARY KEY (name)); CREATE TABLE IF NOT EXISTS domains( - domain VARCHAR(128) NOT NULL, /* if this needs to be larger, InnoDB has a limit of 767 bytes for PRIMARY KEY values! */ + domain VARCHAR(128) NOT NULL UNIQUE, /* if this needs to be larger, InnoDB has a limit of 767 bytes for PRIMARY KEY values! */ zoneName VARCHAR(128) NOT NULL, /* this mostly contains the domain itself again */ configJson TEXT, /* JSON containing the dns backend provider config */ - PRIMARY KEY (domain)); + PRIMARY KEY (domain)) + + /* the default db collation is utf8mb4_unicode_ci but for the app table domain constraint we have to use the old one */ + CHARACTER SET utf8 COLLATE utf8_bin; diff --git a/src/platform.js b/src/platform.js index 0730d831d..d5ec047b7 100644 --- a/src/platform.js +++ b/src/platform.js @@ -12,10 +12,8 @@ var apps = require('./apps.js'), async = require('async'), config = require('./config.js'), certificates = require('./certificates.js'), - DatabaseError = require('./databaseerror.js'), debug = require('debug')('box:platform'), domains = require('./domains.js'), - DomainError = domains.DomainError, fs = require('fs'), hat = require('hat'), infra = require('./infra_version.js'), @@ -25,7 +23,6 @@ var apps = require('./apps.js'), safe = require('safetydance'), semver = require('semver'), settings = require('./settings.js'), - settingsdb = require('./settingsdb.js'), shell = require('./shell.js'), taskmanager = require('./taskmanager.js'), user = require('./user.js'), @@ -67,7 +64,6 @@ function start(callback) { debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version); async.series([ - migrateDNSSettings, stopContainers.bind(null, existingInfra), startAddons.bind(null, existingInfra), removeOldImages, @@ -381,24 +377,3 @@ function startApps(existingInfra, callback) { apps.configureInstalledApps(callback); } } - -// This is only used when updating from single to multi domain support -// REMOVE later! -function migrateDNSSettings(callback) { - assert.strictEqual(typeof callback, 'function'); - - settingsdb.get('dns_config', function (error, result) { - if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error); - - const dnsConfig = result ? safe.JSON.parse(result) : { provider: 'manual' }; - - domains.get(config.fqdn(), function (error, result) { - if (error && error.reason !== DomainError.NOT_FOUND) return callback(error); - - // if domain is already in the table, nothing to do - if (result) return callback(null); - - domains.add(config.fqdn(), config.zoneName(), dnsConfig, callback); - }); - }); -}