diff --git a/src/cloudron.js b/src/cloudron.js index 47f1a1c87..025b2fe0f 100644 --- a/src/cloudron.js +++ b/src/cloudron.js @@ -8,14 +8,13 @@ exports = module.exports = { getConfig: getConfig, getDisks: getDisks, getLogs: getLogs, - getStatus: getStatus, reboot: reboot, isRebootRequired: isRebootRequired, onActivated: onActivated, - setDashboardDns: setDashboardDns, + prepareDashboardDomain: prepareDashboardDomain, setDashboardDomain: setDashboardDomain, renewCerts: renewCerts, @@ -51,16 +50,6 @@ var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'); var NOOP_CALLBACK = function (error) { if (error) debug(error); }; -let gWebadminStatus = { - dns: false, - tls: false, - configuring: false, - restore: { - active: false, - error: null - } -}; - function CloudronError(reason, errorOrMessage) { assert.strictEqual(typeof reason, 'string'); assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined'); @@ -281,37 +270,14 @@ function getLogs(unit, options, callback) { return callback(null, transformStream); } -function getStatus(callback) { - assert.strictEqual(typeof callback, 'function'); - - users.isActivated(function (error, activated) { - if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error)); - - settings.getCloudronName(function (error, cloudronName) { - if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error)); - - callback(null, { - version: config.version(), - apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool - provider: config.provider(), - cloudronName: cloudronName, - adminFqdn: config.adminDomain() ? config.adminFqdn() : null, - activated: activated, - edition: config.edition(), - webadminStatus: gWebadminStatus // only valid when !activated - }); - }); - }); -} - -function setDashboardDns(domain, auditSource, callback) { +function prepareDashboardDomain(domain, auditSource, callback) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof callback, 'function'); - debug(`setDashboardDns: ${domain}`); + debug(`prepareDashboardDomain: ${domain}`); - let task = tasks.startTask(tasks.TASK_DASHBOARD_DNS, [ domain, auditSource ]); + let task = tasks.startTask(tasks.TASK_PREPARE_DASHBOARD_DOMAIN, [ domain, auditSource ]); task.on('error', (error) => callback(new CloudronError(CloudronError.INTERNAL_ERROR, error))); task.on('start', (taskId) => callback(null, taskId)); } diff --git a/src/domains.js b/src/domains.js index 0e66bb32a..3f23b4d92 100644 --- a/src/domains.js +++ b/src/domains.js @@ -26,7 +26,7 @@ module.exports = exports = { parentDomain: parentDomain, - setDashboardDnsRecord: setDashboardDnsRecord, + prepareDashboardDomain: prepareDashboardDomain, DomainsError: DomainsError, @@ -498,7 +498,7 @@ function makeWildcard(hostname) { return parts.join('.'); } -function setDashboardDnsRecord(domain, auditSource, progressCallback, callback) { +function prepareDashboardDomain(domain, auditSource, progressCallback, callback) { assert.strictEqual(typeof domain, 'string'); assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof progressCallback, 'function'); @@ -511,8 +511,11 @@ function setDashboardDnsRecord(domain, auditSource, progressCallback, callback) if (error) return callback(new DomainsError(DomainsError.EXTERNAL_ERROR, error.message)); async.series([ + (done) => { progressCallback({ message: 'Updating DNS' }); done(); }, upsertDnsRecords.bind(null, constants.ADMIN_LOCATION, domain, 'A', [ ip ]), + (done) => { progressCallback({ message: 'Waiting for DNS' }); done(); }, waitForDnsRecord.bind(null, constants.ADMIN_LOCATION, domain, 'A', ip, { interval: 30000, times: 50000 }), + (done) => { progressCallback({ message: 'Getting certificate' }); done(); }, reverseProxy.ensureCertificate.bind(null, fqdn(constants.ADMIN_LOCATION, domainObject), domain, auditSource) ], callback); }); diff --git a/src/provision.js b/src/provision.js index d491c3088..08fe96390 100644 --- a/src/provision.js +++ b/src/provision.js @@ -4,6 +4,7 @@ exports = module.exports = { setup: setup, restore: restore, activate: activate, + getStatus: getStatus, ProvisionError: ProvisionError }; @@ -21,21 +22,32 @@ var assert = require('assert'), DomainsError = domains.DomainsError, eventlog = require('./eventlog.js'), mail = require('./mail.js'), - path = require('path'), safe = require('safetydance'), semver = require('semver'), settingsdb = require('./settingsdb.js'), settings = require('./settings.js'), - shell = require('./shell.js'), superagent = require('superagent'), users = require('./users.js'), UsersError = users.UsersError, tld = require('tldjs'), - util = require('util'); + util = require('util'), + _ = require('underscore'); -var RESTART_CMD = path.join(__dirname, 'scripts/restart.sh'); +const NOOP_CALLBACK = function (error) { if (error) debug(error); }; -var NOOP_CALLBACK = function (error) { if (error) debug(error); }; +// we cannot use tasks since the tasks table gets overwritten when db is imported +let gProvisionStatus = { + setup: { + active: false, + message: '', + errorMessage: null + }, + restore: { + active: false, + message: '', + errorMessage: null + } +}; function ProvisionError(reason, errorOrMessage) { assert.strictEqual(typeof reason, 'string'); @@ -63,6 +75,11 @@ ProvisionError.INTERNAL_ERROR = 'Internal Error'; ProvisionError.EXTERNAL_ERROR = 'External Error'; ProvisionError.ALREADY_PROVISIONED = 'Already Provisioned'; +function setProgress(task, message, callback) { + gProvisionStatus[task].message = message; + callback(); +} + function autoprovision(autoconf, callback) { assert.strictEqual(typeof autoconf, 'object'); assert.strictEqual(typeof callback, 'function'); @@ -102,7 +119,7 @@ function unprovision(callback) { config.setAdminDomain(''); config.setAdminFqdn(''); - config.setAdminLocation('my'); + config.setAdminLocation(constants.ADMIN_LOCATION); // TODO: also cancel any existing configureWebadmin task async.series([ @@ -117,23 +134,27 @@ function setup(dnsConfig, autoconf, auditSource, callback) { assert.strictEqual(typeof auditSource, 'object'); assert.strictEqual(typeof callback, 'function'); + if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) return callback(new ProvisionError(ProvisionError.BAD_STATE, 'Already setting up or restoring')); + + gProvisionStatus.setup = { active: true, errorMessage: '', message: 'Adding domain' }; + + function done(error) { + gProvisionStatus.setup.active = false; + gProvisionStatus.setup.errorMessage = error ? error.message : ''; + callback(error); + } + users.isActivated(function (error, activated) { - if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); - if (activated) return callback(new ProvisionError(ProvisionError.ALREADY_SETUP)); + if (error) return done(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + if (activated) return done(new ProvisionError(ProvisionError.ALREADY_SETUP)); unprovision(function (error) { - if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); - - let webadminStatus = cloudron.getWebadminStatus(); - - if (webadminStatus.configuring || webadminStatus.restore.active) return callback(new ProvisionError(ProvisionError.BAD_STATE, 'Already restoring or configuring')); + if (error) return done(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); const domain = dnsConfig.domain.toLowerCase(); const zoneName = dnsConfig.zoneName ? dnsConfig.zoneName : (tld.getDomain(domain) || domain); - const adminFqdn = 'my' + (dnsConfig.config.hyphenatedSubdomains ? '-' : '.') + domain; - - debug(`provision: Setting up Cloudron with domain ${domain} and zone ${zoneName} using admin fqdn ${adminFqdn}`); + debug(`provision: Setting up Cloudron with domain ${domain} and zone ${zoneName}`); let data = { zoneName: zoneName, @@ -144,17 +165,24 @@ function setup(dnsConfig, autoconf, auditSource, callback) { }; domains.add(domain, data, auditSource, function (error) { - if (error && error.reason === DomainsError.BAD_FIELD) return callback(new ProvisionError(ProvisionError.BAD_FIELD, error.message)); - if (error && error.reason === DomainsError.ALREADY_EXISTS) return callback(new ProvisionError(ProvisionError.BAD_FIELD, error.message)); - if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + if (error && error.reason === DomainsError.BAD_FIELD) return done(new ProvisionError(ProvisionError.BAD_FIELD, error.message)); + if (error && error.reason === DomainsError.ALREADY_EXISTS) return done(new ProvisionError(ProvisionError.BAD_FIELD, error.message)); + if (error) return done(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + + callback(); // now that args are validated run the task in the background async.series([ mail.addDomain.bind(null, domain), - cloudron.setDashboardDns.bind(null, domain, auditSource), + domains.prepareDashboardDomain.bind(null, domain, auditSource, (progress) => setProgress('setup', progress.message, NOOP_CALLBACK)), cloudron.setDashboardDomain.bind(null, domain), + setProgress.bind(null, 'setup', 'Applying auto-configuration'), autoprovision.bind(null, autoconf), + setProgress.bind(null, 'setup', 'Done'), eventlog.add.bind(null, eventlog.ACTION_PROVISION, auditSource, { }) - ], callback); + ], function (error) { + gProvisionStatus.setup.active = false; + gProvisionStatus.setup.errorMessage = error ? error.message : ''; + }); }); }); }); @@ -230,40 +258,67 @@ function restore(backupConfig, backupId, version, autoconf, auditSource, callbac if (!semver.valid(version)) return callback(new ProvisionError(ProvisionError.BAD_STATE, 'version is not a valid semver')); if (semver.major(config.version()) !== semver.major(version) || semver.minor(config.version()) !== semver.minor(version)) return callback(new ProvisionError(ProvisionError.BAD_STATE, `Run cloudron-setup with --version ${version} to restore from this backup`)); - let webadminStatus = cloudron.getWebadminStatus(); + if (gProvisionStatus.setup.active || gProvisionStatus.restore.active) return callback(new ProvisionError(ProvisionError.BAD_STATE, 'Already setting up or restoring')); - if (webadminStatus.configuring || webadminStatus.restore.active) return callback(new ProvisionError(ProvisionError.BAD_STATE, 'Already restoring or configuring')); + gProvisionStatus.restore = { active: true, errorMessage: '', message: 'Testing backup config' }; + + function done(error) { + gProvisionStatus.restore.active = false; + gProvisionStatus.restore.errorMessage = error ? error.message : ''; + callback(error); + } users.isActivated(function (error, activated) { - if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); - if (activated) return callback(new ProvisionError(ProvisionError.ALREADY_PROVISIONED, 'Already activated')); + if (error) return done(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + if (activated) return done(new ProvisionError(ProvisionError.ALREADY_PROVISIONED, 'Already activated')); backups.testConfig(backupConfig, function (error) { - if (error && error.reason === BackupsError.BAD_FIELD) return callback(new ProvisionError(ProvisionError.BAD_FIELD, error.message)); - if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new ProvisionError(ProvisionError.EXTERNAL_ERROR, error.message)); - if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + if (error && error.reason === BackupsError.BAD_FIELD) return done(new ProvisionError(ProvisionError.BAD_FIELD, error.message)); + if (error && error.reason === BackupsError.EXTERNAL_ERROR) return done(new ProvisionError(ProvisionError.EXTERNAL_ERROR, error.message)); + if (error) return done(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); debug(`restore: restoring from ${backupId} from provider ${backupConfig.provider} with format ${backupConfig.format}`); - webadminStatus.restore.active = true; - webadminStatus.restore.error = null; - - callback(null); // do no block + callback(); // now that the fields are validated, continue task in the background async.series([ - backups.restore.bind(null, backupConfig, backupId, (progress) => debug(`restore: ${progress}`)), - eventlog.add.bind(null, eventlog.ACTION_RESTORE, auditSource, { backupId }), + setProgress.bind(null, 'restore', 'Downloading backup'), + backups.restore.bind(null, backupConfig, backupId, (progress) => setProgress('restore', progress.message, NOOP_CALLBACK)), + setProgress.bind(null, 'restore', 'Applying auto-configuration'), autoprovision.bind(null, autoconf), // currently, our suggested restore flow is after a dnsSetup. The dnSetup creates DKIM keys and updates the DNS // for this reason, we have to re-setup DNS after a restore so it has DKIm from the backup // Once we have a 100% IP based restore, we can skip this mail.setDnsRecords.bind(null, config.adminDomain()), - shell.sudo.bind(null, 'restart', [ RESTART_CMD ], {}) + eventlog.add.bind(null, eventlog.ACTION_RESTORE, auditSource, { backupId }), ], function (error) { - debug('restore:', error); - if (error) webadminStatus.restore.error = error.message; - webadminStatus.restore.active = false; + gProvisionStatus.restore.active = false; + gProvisionStatus.restore.errorMessage = error ? error.message : ''; + + if (!error) cloudron.onActivated(NOOP_CALLBACK); }); }); }); } + +function getStatus(callback) { + assert.strictEqual(typeof callback, 'function'); + + users.isActivated(function (error, activated) { + if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + + settings.getCloudronName(function (error, cloudronName) { + if (error) return callback(new ProvisionError(ProvisionError.INTERNAL_ERROR, error)); + + callback(null, _.extend({ + version: config.version(), + apiServerOrigin: config.apiServerOrigin(), // used by CaaS tool + provider: config.provider(), + cloudronName: cloudronName, + adminFqdn: config.adminDomain() ? config.adminFqdn() : null, + activated: activated, + edition: config.edition() + }, gProvisionStatus)); + }); + }); +} diff --git a/src/routes/cloudron.js b/src/routes/cloudron.js index 22154c502..23f20974d 100644 --- a/src/routes/cloudron.js +++ b/src/routes/cloudron.js @@ -11,9 +11,8 @@ exports = module.exports = { checkForUpdates: checkForUpdates, getLogs: getLogs, getLogStream: getLogStream, - getStatus: getStatus, setDashboardDomain: setDashboardDomain, - setDashboardDns: setDashboardDns, + prepareDashboardDomain: prepareDashboardDomain, renewCerts: renewCerts }; @@ -187,10 +186,10 @@ function setDashboardDomain(req, res, next) { }); } -function setDashboardDns(req, res, next) { +function prepareDashboardDomain(req, res, next) { if (!req.body.domain || typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string')); - cloudron.setDashboardDns(req.body.domain, auditSource(req), function (error, taskId) { + cloudron.prepareDashboardDomain(req.body.domain, auditSource(req), function (error, taskId) { if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(404, error.message)); if (error) return next(new HttpError(500, error)); @@ -198,14 +197,6 @@ function setDashboardDns(req, res, next) { }); } -function getStatus(req, res, next) { - cloudron.getStatus(function (error, status) { - if (error) return next(new HttpError(500, error)); - - next(new HttpSuccess(200, status)); - }); -} - function renewCerts(req, res, next) { cloudron.renewCerts({ domain: req.body.domain || null }, auditSource(req), function (error, taskId) { if (error && error.reason === CloudronError.NOT_FOUND) return next(new HttpError(404, error.message)); diff --git a/src/routes/provision.js b/src/routes/provision.js index ff14e5c26..6dfa3a67a 100644 --- a/src/routes/provision.js +++ b/src/routes/provision.js @@ -5,7 +5,8 @@ exports = module.exports = { setupTokenAuth: setupTokenAuth, setup: setup, activate: activate, - restore: restore + restore: restore, + getStatus: getStatus }; var assert = require('assert'), @@ -156,3 +157,11 @@ function restore(req, res, next) { next(new HttpSuccess(200)); }); } + +function getStatus(req, res, next) { + provision.getStatus(function (error, status) { + if (error) return next(new HttpError(500, error)); + + next(new HttpSuccess(200, status)); + }); +} diff --git a/src/routes/test/mail-test.js b/src/routes/test/mail-test.js index 3dfefe744..fb4eb1d7c 100644 --- a/src/routes/test/mail-test.js +++ b/src/routes/test/mail-test.js @@ -48,7 +48,7 @@ function setup(done) { function dnsSetup(callback) { superagent.post(SERVER_URL + '/api/v1/cloudron/setup') - .send({ dnsConfig: { provider: ADMIN_DOMAIN.provider, domain: ADMIN_DOMAIN.domain, config: ADMIN_DOMAIN.config } }) + .send({ dnsConfig: { provider: ADMIN_DOMAIN.provider, domain: ADMIN_DOMAIN.domain, config: ADMIN_DOMAIN.config, tlsConfig: { provider: 'fallback' } } }) .end(function (error, result) { expect(result).to.be.ok(); expect(result.statusCode).to.eql(200); @@ -57,6 +57,21 @@ function setup(done) { }); }, + function waitForSetup(done) { + async.retry({ times: 5, interval: 4000 }, function (retryCallback) { + superagent.get(SERVER_URL + '/api/v1/cloudron/status') + .end(function (error, result) { + if (!result || result.statusCode !== 200) return retryCallback(new Error('Bad result')); + + console.dir(result.body); + + if (!result.body.setup.active && result.body.setup.errorMessage === '' && result.body.adminFqdn) return retryCallback(); + + retryCallback(new Error('Not done yet: ' + JSON.stringify(result.body))); + }); + }, done); + }, + function createAdmin(callback) { superagent.post(SERVER_URL + '/api/v1/cloudron/activate') .query({ setupToken: 'somesetuptoken' }) diff --git a/src/routes/test/server-test.js b/src/routes/test/server-test.js index c53cec95c..25c279843 100644 --- a/src/routes/test/server-test.js +++ b/src/routes/test/server-test.js @@ -34,6 +34,19 @@ function cleanup(done) { ], done); } +function waitForSetup(done) { + async.retry({ times: 5, interval: 4000 }, function (retryCallback) { + superagent.get(SERVER_URL + '/api/v1/cloudron/status') + .end(function (error, result) { + if (!result || result.statusCode !== 200) return retryCallback(new Error('Bad result')); + + if (!result.body.setup.active && result.body.setup.errorMessage === '' && result.body.adminFqdn) return retryCallback(); + + retryCallback(new Error('Not done yet: ' + JSON.stringify(result.body))); + }); + }, done); +} + describe('REST API', function () { before(setup); after(cleanup); @@ -128,23 +141,23 @@ describe('REST API', function () { it('dns setup succeeds', function (done) { superagent.post(SERVER_URL + '/api/v1/cloudron/setup') - .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {} } }) + .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, adminFqdn: 'my.' + DOMAIN, config: {}, tlsConfig: { provider: 'fallback' } } }) .end(function (error, result) { expect(result).to.be.ok(); expect(result.statusCode).to.eql(200); - done(); + waitForSetup(done); }); }); it('dns setup twice succeeds', function (done) { superagent.post(SERVER_URL + '/api/v1/cloudron/setup') - .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, DOMAIN, config: {} } }) + .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, DOMAIN, config: {} }, tlsConfig: { provider: 'fallback' } }) .end(function (error, result) { expect(result).to.be.ok(); expect(result.statusCode).to.eql(200); - done(); + waitForSetup(done); }); }); diff --git a/src/server.js b/src/server.js index d9a54181a..32134f8d2 100644 --- a/src/server.js +++ b/src/server.js @@ -111,7 +111,7 @@ function initializeExpressSync() { router.post('/api/v1/cloudron/setup', routes.provision.providerTokenAuth, routes.provision.setup); // only available until no-domain router.post('/api/v1/cloudron/restore', routes.provision.restore); // only available until activated router.post('/api/v1/cloudron/activate', routes.provision.setupTokenAuth, routes.provision.activate); - router.get ('/api/v1/cloudron/status', routes.cloudron.getStatus); + router.get ('/api/v1/cloudron/status', routes.provision.getStatus); router.get ('/api/v1/cloudron/avatar', routes.settings.getCloudronAvatar); // this is a public alias for /api/v1/settings/cloudron_avatar @@ -121,7 +121,7 @@ function initializeExpressSync() { // cloudron routes router.get ('/api/v1/cloudron/update', cloudronScope, routes.cloudron.getUpdateInfo); router.post('/api/v1/cloudron/update', cloudronScope, routes.cloudron.update); - router.post('/api/v1/cloudron/set_dashboard_dns', cloudronScope, routes.cloudron.setDashboardDns); + router.post('/api/v1/cloudron/prepare_dashboard_dns', cloudronScope, routes.cloudron.prepareDashboardDomain); router.post('/api/v1/cloudron/set_dashboard_domain', cloudronScope, routes.cloudron.setDashboardDomain); router.post('/api/v1/cloudron/renew_certs', cloudronScope, routes.cloudron.renewCerts); router.post('/api/v1/cloudron/check_for_updates', cloudronScope, routes.cloudron.checkForUpdates); diff --git a/src/tasks.js b/src/tasks.js index df8e6de5f..e2262b4e7 100644 --- a/src/tasks.js +++ b/src/tasks.js @@ -18,7 +18,7 @@ exports = module.exports = { TASK_BACKUP: 'backup', TASK_UPDATE: 'update', TASK_RENEW_CERTS: 'renewcerts', - TASK_DASHBOARD_DNS: 'dashboardDns', + TASK_PREPARE_DASHBOARD_DOMAIN: 'prepareDashboardDomain', // testing _TASK_IDENTITY: '_identity', diff --git a/src/taskworker.js b/src/taskworker.js index c20b520e8..d1c96f9a4 100755 --- a/src/taskworker.js +++ b/src/taskworker.js @@ -17,7 +17,7 @@ const TASKS = { // indexed by task type backup: backups.backupBoxAndApps, update: updater.update, renewcerts: reverseProxy.renewCerts, - dashboardDns: domains.setDashboardDnsRecord, + prepareDashboardDomain: domains.prepareDashboardDomain, _identity: (arg, progressCallback, callback) => callback(null, arg), _error: (arg, progressCallback, callback) => callback(new Error(`Failed for arg: ${arg}`)),