diff --git a/package-lock.json b/package-lock.json index 6cc654c53..b444b1b89 100644 --- a/package-lock.json +++ b/package-lock.json @@ -638,6 +638,13 @@ "semver": "^7.3.5", "tv4": "^1.3.0", "validator": "^13.6.0" + }, + "dependencies": { + "safetydance": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-2.0.1.tgz", + "integrity": "sha512-RuMpDOXn4bC+cIrzeZ6ZJR8/aaa+58KztATWO8KbEpfC4LRaYskn+Ll3H5KMikH1N1F47S+razKDqZklUkRkTg==" + } } }, "code-point-at": { @@ -3972,6 +3979,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -3992,9 +4000,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "safetydance": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-2.0.1.tgz", - "integrity": "sha512-RuMpDOXn4bC+cIrzeZ6ZJR8/aaa+58KztATWO8KbEpfC4LRaYskn+Ll3H5KMikH1N1F47S+razKDqZklUkRkTg==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/safetydance/-/safetydance-2.1.0.tgz", + "integrity": "sha512-N1/afVgkNv39mBF2HzQEVt6XqWlgEIefFd87OBOiXS+9wRuCS/xoXzuqdMdxlkfYL2sFd7mqnRivFDPfl59nTA==" }, "sass-graph": { "version": "2.2.5", diff --git a/package.json b/package.json index afde932ac..f68232a74 100644 --- a/package.json +++ b/package.json @@ -56,9 +56,8 @@ "qrcode": "^1.4.4", "readdirp": "^3.6.0", "request": "^2.88.2", - "rimraf": "^3.0.2", "s3-block-read-stream": "^0.5.0", - "safetydance": "^2.0.1", + "safetydance": "^2.1.0", "semver": "^7.3.5", "speakeasy": "^2.0.0", "split": "^1.0.1", diff --git a/src/apphealthmonitor.js b/src/apphealthmonitor.js index 8a471211a..8ab964bdf 100644 --- a/src/apphealthmonitor.js +++ b/src/apphealthmonitor.js @@ -2,7 +2,6 @@ const apps = require('./apps.js'), assert = require('assert'), - async = require('async'), auditSource = require('./auditsource.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), @@ -10,24 +9,21 @@ const apps = require('./apps.js'), docker = require('./docker.js'), eventlog = require('./eventlog.js'), safe = require('safetydance'), - superagent = require('superagent'), - util = require('util'); + superagent = require('superagent'); exports = module.exports = { run }; -const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable const UNHEALTHY_THRESHOLD = 20 * 60 * 1000; // 20 minutes const OOM_EVENT_LIMIT = 60 * 60 * 1000; // will only raise 1 oom event every hour let gStartTime = null; // time when apphealthmonitor was started let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago -function setHealth(app, health, callback) { +async function setHealth(app, health) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof health, 'string'); - assert.strictEqual(typeof callback, 'function'); // app starts out with null health // if it became healthy, we update immediately. this is required for ui to say "running" etc @@ -53,68 +49,60 @@ function setHealth(app, health, callback) { } } else { debug(`setHealth: ${app.id} (${app.fqdn}) waiting for ${(UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000} to update health`); - return callback(null); + return; } - util.callbackify(apps.setHealth)(app.id, health, healthTime, function (error) { - if (error && error.reason === BoxError.NOT_FOUND) return callback(null); // app uninstalled? - if (error) return callback(error); + const [error] = await safe(apps.setHealth(app.id, health, healthTime)); + if (error && error.reason === BoxError.NOT_FOUND) return; // app uninstalled? + if (error) throw error; - app.health = health; - app.healthTime = healthTime; - - callback(null); - }); + app.health = health; + app.healthTime = healthTime; } // callback is called with error for fatal errors and not if health check failed -function checkAppHealth(app, callback) { +async function checkAppHealth(app, options) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); + assert.strictEqual(typeof options, 'object'); - if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) { - return callback(null); - } + if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) return; const manifest = app.manifest; - docker.inspect(app.containerId, function (error, data) { - if (error || !data || !data.State) return setHealth(app, apps.HEALTH_ERROR, callback); - if (data.State.Running !== true) return setHealth(app, apps.HEALTH_DEAD, callback); + const [error, data] = await safe(docker.inspect(app.containerId)); + if (error || !data || !data.State) return await setHealth(app, apps.HEALTH_ERROR); + if (data.State.Running !== true) return await setHealth(app, apps.HEALTH_DEAD); - // non-appstore apps may not have healthCheckPath - if (!manifest.healthCheckPath) return setHealth(app, apps.HEALTH_HEALTHY, callback); + // non-appstore apps may not have healthCheckPath + if (!manifest.healthCheckPath) return await setHealth(app, apps.HEALTH_HEALTHY); - const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`; - superagent - .get(healthCheckUrl) - .set('Host', app.fqdn) // required for some apache configs with rewrite rules - .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio) - .redirects(0) - .timeout(HEALTHCHECK_INTERVAL) - .end(function (error, res) { - if (error && !error.response) { - setHealth(app, apps.HEALTH_UNHEALTHY, callback); - } else if (res.statusCode > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites) - setHealth(app, apps.HEALTH_UNHEALTHY, callback); - } else { - setHealth(app, apps.HEALTH_HEALTHY, callback); - } - }); - }); + const healthCheckUrl = `http://${app.containerIp}:${manifest.httpPort}${manifest.healthCheckPath}`; + const [healthCheckError, response] = await safe(superagent + .get(healthCheckUrl) + .set('Host', app.fqdn) // required for some apache configs with rewrite rules + .set('User-Agent', 'Mozilla (CloudronHealth)') // required for some apps (e.g. minio) + .redirects(0) + .ok(() => true) + .timeout(options.timeout * 1000)); + + if (healthCheckError) { + await setHealth(app, apps.HEALTH_UNHEALTHY); + } else if (response.status > 403) { // 2xx and 3xx are ok. even 401 and 403 are ok for now (for WP sites) + await setHealth(app, apps.HEALTH_UNHEALTHY); + } else { + await setHealth(app, apps.HEALTH_HEALTHY); + } } -function getContainerInfo(containerId, callback) { - docker.inspect(containerId, function (error, result) { - if (error) return callback(error); +async function getContainerInfo(containerId) { + const result = await docker.inspect(containerId); - const appId = safe.query(result, 'Config.Labels.appId', null); + const appId = safe.query(result, 'Config.Labels.appId', null); - if (!appId) return callback(null, null /* app */, { name: result.Name.slice(1) }); // addon . Name has a '/' in the beginning for some reason + if (!appId) return { app: null, addon: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason - util.callbackify(apps.get)(appId, callback); // don't get by container id as this can be an exec container - }); + return await apps.get(appId); // don't get by container id as this can be an exec container } /* @@ -122,83 +110,68 @@ function getContainerInfo(containerId, callback) { docker run -ti -m 100M cloudron/base:3.0.0 /bin/bash stress --vm 1 --vm-bytes 200M --vm-hang 0 */ -function processDockerEvents(intervalSecs, callback) { - assert.strictEqual(typeof intervalSecs, 'number'); - assert.strictEqual(typeof callback, 'function'); +async function processDockerEvents(options) { + assert.strictEqual(typeof options, 'object'); - const since = ((new Date().getTime() / 1000) - intervalSecs).toFixed(0); + const since = ((new Date().getTime() / 1000) - options.intervalSecs).toFixed(0); const until = ((new Date().getTime() / 1000) - 1).toFixed(0); - docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }, function (error, stream) { - if (error) return callback(error); + const stream = await docker.getEvents({ since: since, until: until, filters: JSON.stringify({ event: [ 'oom' ] }) }); + stream.setEncoding('utf8'); + stream.on('data', async function (data) { + const event = JSON.parse(data); + const containerId = String(event.id); - stream.setEncoding('utf8'); - stream.on('data', function (data) { - const event = JSON.parse(data); - const containerId = String(event.id); + const [error, info] = await safe(getContainerInfo(containerId)); + const program = error ? containerId : (info.app ? info.app.fqdn : info.addon.name); + const now = Date.now(); + const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT); - getContainerInfo(containerId, function (error, app, addon) { - const program = error ? containerId : (app ? app.fqdn : addon.name); - const now = Date.now(); - const notifyUser = !(app && app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT); + debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`); - debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`); + // do not send mails for dev apps + if (notifyUser) { + // app can be null for addon containers + await eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: info.addon || null, app: info.app || null }); - // do not send mails for dev apps - if (notifyUser) { - // app can be null for addon containers - eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: addon || null, app: app || null }); - - gLastOomMailTime = now; - } - }); - }); - - stream.on('error', function (error) { - debug('Error reading docker events', error); - callback(); - }); - - stream.on('end', callback); - - // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time) - setTimeout(stream.destroy.bind(stream), 3000); // https://github.com/apocas/dockerode/issues/179 + gLastOomMailTime = now; + } }); + + stream.on('error', function (error) { + debug('Error reading docker events', error); + }); + + stream.on('end', function () { + // debug('Event stream ended'); + }); + + // safety hatch if 'until' doesn't work (there are cases where docker is working with a different time) + setTimeout(stream.destroy.bind(stream), options.timeout); // https://github.com/apocas/dockerode/issues/179 } -function processApp(callback) { - assert.strictEqual(typeof callback, 'function'); +async function processApp(options) { + assert.strictEqual(typeof options, 'object'); - const appsList = util.callbackify(apps.list); + const allApps = await apps.list(); - appsList(function (error, allApps) { - if (error) return callback(error); + const healthChecks = allApps.map((app) => checkAppHealth(app, options)); // start healthcheck in parallel - async.each(allApps, checkAppHealth, function (error) { - const alive = allApps - .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; }); + await Promise.allSettled(healthChecks); // wait for all promises to finish - debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : '')); + const alive = allApps + .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; }); - callback(null); - }); - }); + debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.`); } -function run(intervalSecs, callback) { +async function run(intervalSecs) { assert.strictEqual(typeof intervalSecs, 'number'); - assert.strictEqual(typeof callback, 'function'); if (constants.TEST) return; if (!gStartTime) gStartTime = new Date(); - async.series([ - processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes - processDockerEvents.bind(null, intervalSecs) - ], function (error) { - if (error) debug(`run: could not check app health. ${error.message}`); - - callback(); - }); + await processApp({ timeout: (intervalSecs - 3) * 1000 }); + await processDockerEvents({ intervalSecs, timeout: 3000 }); } diff --git a/src/apps.js b/src/apps.js index 6c9710f05..334d0ac96 100644 --- a/src/apps.js +++ b/src/apps.js @@ -2019,19 +2019,18 @@ function checkManifestConstraints(manifest) { return null; } -function exec(app, options, callback) { +async function exec(app, options) { assert.strictEqual(typeof app, 'object'); assert(options && typeof options === 'object'); - assert.strictEqual(typeof callback, 'function'); - let cmd = options.cmd || [ '/bin/bash' ]; + const cmd = options.cmd || [ '/bin/bash' ]; assert(Array.isArray(cmd) && cmd.length > 0); if (app.installationState !== exports.ISTATE_INSTALLED || app.runState !== exports.RSTATE_RUNNING) { - return callback(new BoxError(BoxError.BAD_STATE, 'App not installed or running')); + throw new BoxError(BoxError.BAD_STATE, 'App not installed or running'); } - var execOptions = { + const execOptions = { AttachStdin: true, AttachStdout: true, AttachStderr: true, @@ -2043,7 +2042,7 @@ function exec(app, options, callback) { Cmd: cmd }; - var startOptions = { + const startOptions = { Detach: false, Tty: options.tty, // hijacking upgrades the docker connection from http to tcp. because of this upgrade, @@ -2058,12 +2057,8 @@ function exec(app, options, callback) { stderr: true }; - docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns }, function (error, stream) { - if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running - if (error) return callback(error); - - callback(null, stream); - }); + const stream = await docker.execContainer(app.containerId, { execOptions, startOptions, rows: options.rows, columns: options.columns }); + return stream; } function canAutoupdateApp(app, updateInfo) { @@ -2205,8 +2200,6 @@ async function configureInstalledApps() { async function restartAppsUsingAddons(changedAddons) { assert(Array.isArray(changedAddons)); - const stopContainers = util.promisify(docker.stopContainers); - let apps = await list(); apps = apps.filter(app => app.manifest.addons && _.intersection(Object.keys(app.manifest.addons), changedAddons).length !== 0); apps = apps.filter(app => app.installationState !== exports.ISTATE_ERROR); // remove errored apps. let them be 'repaired' by hand @@ -2222,7 +2215,7 @@ async function restartAppsUsingAddons(changedAddons) { }; // stop apps before updating the databases because postgres will "lock" them preventing import - const [stopError] = await safe(stopContainers(app.id)); + const [stopError] = await safe(docker.stopContainers(app.id)); if (stopError) debug(`restartAppsUsingAddons: error stopping ${app.fqdn}`, stopError); const [addTaskError, taskId] = await safe(addTask(app.id, exports.ISTATE_PENDING_RESTART, task)); @@ -2235,7 +2228,7 @@ async function restartAppsUsingAddons(changedAddons) { async function schedulePendingTasks() { debug('schedulePendingTasks: scheduling app tasks'); - const result = list(); + const result = await list(); result.forEach(function (app) { if (!app.taskId) return; // if not in any pending state, do nothing diff --git a/src/apptask.js b/src/apptask.js index d0501987e..e0942b69f 100644 --- a/src/apptask.js +++ b/src/apptask.js @@ -31,8 +31,8 @@ const apps = require('./apps.js'), os = require('os'), path = require('path'), paths = require('./paths.js'), + promiseRetry = require('./promise-retry.js'), reverseProxy = require('./reverseproxy.js'), - rimraf = require('rimraf'), safe = require('safetydance'), services = require('./services.js'), settings = require('./settings.js'), @@ -64,54 +64,41 @@ function makeTaskError(error, app) { } // updates the app object and the database -function updateApp(app, values, callback) { +async function updateApp(app, values) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof values, 'object'); - assert.strictEqual(typeof callback, 'function'); - util.callbackify(apps.update)(app.id, values, function (error) { - if (error) return callback(error); + await apps.update(app.id, values); - for (var value in values) { - app[value] = values[value]; - } - - callback(null); - }); + for (const value in values) { + app[value] = values[value]; + } } -function allocateContainerIp(app, callback) { +async function allocateContainerIp(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); - async.retry({ times: 10 }, function (retryCallback) { + await promiseRetry({ times: 10, interval: 0}, async function () { const iprange = iputils.intFromIp('172.18.20.255') - iputils.intFromIp('172.18.16.1'); let rnd = Math.floor(Math.random() * iprange); const containerIp = iputils.ipFromInt(iputils.intFromIp('172.18.16.1') + rnd); - updateApp(app, { containerIp }, retryCallback); - }, callback); + updateApp(app, { containerIp }); + }); } -function createContainer(app, callback) { +async function createContainer(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); assert(!app.containerId); // otherwise, it will trigger volumeFrom debugApp(app, 'creating container'); - docker.createContainer(app, function (error, container) { - if (error) return callback(error); + const container = await docker.createContainer(app); - updateApp(app, { containerId: container.id }, function (error) { - if (error) return callback(error); + await updateApp(app, { containerId: container.id }); - // re-generate configs that rely on container id - async.series([ - addLogrotateConfig.bind(null, app), - addCollectdProfile.bind(null, app) - ], callback); - }); - }); + // re-generate configs that rely on container id + await addLogrotateConfig(app); + await addCollectdProfile(app); } function deleteContainers(app, options, callback) { @@ -197,40 +184,30 @@ async function removeCollectdProfile(app) { await collectd.removeProfile(app.id); } -function addLogrotateConfig(app, callback) { +async function addLogrotateConfig(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); - docker.inspect(app.containerId, function (error, result) { - if (error) return callback(error); + const result = await docker.inspect(app.containerId); - var runVolume = result.Mounts.find(function (mount) { return mount.Destination === '/run'; }); - if (!runVolume) return callback(new BoxError(BoxError.DOCKER_ERROR, 'App does not have /run mounted')); + const runVolume = result.Mounts.find(function (mount) { return mount.Destination === '/run'; }); + if (!runVolume) throw new BoxError(BoxError.DOCKER_ERROR, 'App does not have /run mounted'); - // logrotate configs can have arbitrary commands, so the config files must be owned by root - var logrotateConf = ejs.render(LOGROTATE_CONFIG_EJS, { volumePath: runVolume.Source, appId: app.id }); - var tmpFilePath = path.join(os.tmpdir(), app.id + '.logrotate'); - fs.writeFile(tmpFilePath, logrotateConf, function (error) { - if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error writing logrotate config: ${error.message}`)); + // logrotate configs can have arbitrary commands, so the config files must be owned by root + const logrotateConf = ejs.render(LOGROTATE_CONFIG_EJS, { volumePath: runVolume.Source, appId: app.id }); + const tmpFilePath = path.join(os.tmpdir(), app.id + '.logrotate'); - shell.sudo('addLogrotateConfig', [ CONFIGURE_LOGROTATE_CMD, 'add', app.id, tmpFilePath ], {}, function (error) { - if (error) return callback(new BoxError(BoxError.LOGROTATE_ERROR, `Error adding logrotate config: ${error.message}`)); + safe.fs.writeFileSync(tmpFilePath, logrotateConf); + if (safe.error) throw new BoxError(BoxError.FS_ERROR, `Error writing logrotate config: ${safe.error.message}`); - callback(null); - }); - }); - }); + const [error] = await safe(shell.promises.sudo('addLogrotateConfig', [ CONFIGURE_LOGROTATE_CMD, 'add', app.id, tmpFilePath ], {})); + if (error) throw new BoxError(BoxError.LOGROTATE_ERROR, `Error adding logrotate config: ${error.message}`); } -function removeLogrotateConfig(app, callback) { +async function removeLogrotateConfig(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); - shell.sudo('removeLogrotateConfig', [ CONFIGURE_LOGROTATE_CMD, 'remove', app.id ], {}, function (error) { - if (error) return callback(new BoxError(BoxError.LOGROTATE_ERROR, `Error removing logrotate config: ${error.message}`)); - - callback(null); - }); + const [error] = await safe(shell.promises.sudo('removeLogrotateConfig', [ CONFIGURE_LOGROTATE_CMD, 'remove', app.id ], {})); + if (error) throw new BoxError(BoxError.LOGROTATE_ERROR, `Error removing logrotate config: ${error.message}`); } function cleanupLogs(app, callback) { @@ -238,7 +215,7 @@ function cleanupLogs(app, callback) { assert.strictEqual(typeof callback, 'function'); // note that redis container logs are cleaned up by the addon - rimraf(path.join(paths.LOG_DIR, app.id), function (error) { + fs.rm(path.join(paths.LOG_DIR, app.id), { force: true, recursive: true }, function (error) { if (error) debugApp(app, 'cannot cleanup logs:', error); callback(null); @@ -258,29 +235,27 @@ function verifyManifest(manifest, callback) { callback(null); } -function downloadIcon(app, callback) { +async function downloadIcon(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); // nothing to download if we dont have an appStoreId - if (!app.appStoreId) return callback(null); + if (!app.appStoreId) return; debugApp(app, `Downloading icon of ${app.appStoreId}@${app.manifest.version}`); const iconUrl = settings.apiServerOrigin() + '/api/v1/apps/' + app.appStoreId + '/versions/' + app.manifest.version + '/icon'; - async.retry({ times: 10, interval: 5000 }, function (retryCallback) { - superagent - .get(iconUrl) + await promiseRetry({ times: 10, interval: 5000 }, async function () { + const [networkError, response] = await safe(superagent.get(iconUrl) .buffer(true) .timeout(30 * 1000) - .end(function (error, res) { - if (error && !error.response) return retryCallback(new BoxError(BoxError.NETWORK_ERROR, `Network error downloading icon : ${error.message}`)); - if (res.statusCode !== 200) return retryCallback(null); // ignore error. this can also happen for apps installed with cloudron-cli + .ok(() => true)); - updateApp(app, { appStoreIcon: res.body }, retryCallback); - }); - }, callback); + if (networkError) throw new BoxError(BoxError.NETWORK_ERROR, `Network error downloading icon : ${networkError.message}`); + if (response.status !== 200) return; // ignore error. this can also happen for apps installed with cloudron-cli + + await updateApp(app, { appStoreIcon: response.body }); + }); } function waitForDnsPropagation(app, callback) { @@ -329,33 +304,24 @@ function moveDataDir(app, targetDir, callback) { }); } -function downloadImage(manifest, callback) { +async function downloadImage(manifest) { assert.strictEqual(typeof manifest, 'object'); - assert.strictEqual(typeof callback, 'function'); - docker.info(function (error, info) { - if (error) return callback(error); + const info = await docker.info(); + const [dfError, diskUsage] = await safe(df.file(info.DockerRootDir)); + if (dfError) throw new BoxError(BoxError.FS_ERROR, `Error getting file system info: ${dfError.message}`); - const dfAsync = util.callbackify(df.file); - dfAsync(info.DockerRootDir, function (error, diskUsage) { - if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error getting file system info: ${error.message}`)); - if (diskUsage.available < (1024*1024*1024)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Not enough disk space to pull docker image', { diskUsage: diskUsage, dockerRootDir: info.DockerRootDir })); + if (diskUsage.available < (1024*1024*1024)) throw new BoxError(BoxError.DOCKER_ERROR, 'Not enough disk space to pull docker image', { diskUsage: diskUsage, dockerRootDir: info.DockerRootDir }); - docker.downloadImage(manifest, function (error) { - if (error) return callback(error); - - callback(null); - }); - }); - }); + await docker.downloadImage(manifest); } -function startApp(app, callback){ +async function startApp(app) { debugApp(app, 'startApp: starting container'); - if (app.runState === apps.RSTATE_STOPPED) return callback(); + if (app.runState === apps.RSTATE_STOPPED) return; - docker.startContainer(app.id, callback); + await docker.startContainer(app.id); } function install(app, args, progressCallback, callback) { @@ -378,7 +344,7 @@ function install(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 10, message: 'Cleaning up old install' }), reverseProxy.unconfigureApp.bind(null, app), deleteContainers.bind(null, app, { managedOnly: true }), - function teardownAddons(next) { + async function teardownAddons() { // when restoring, app does not require these addons anymore. remove carefully to preserve the db passwords let addonsToRemove; if (oldManifest) { @@ -387,7 +353,7 @@ function install(app, args, progressCallback, callback) { addonsToRemove = app.manifest.addons; } - services.teardownAddons(app, addonsToRemove, next); + await services.teardownAddons(app, addonsToRemove); }, function deleteAppDirIfNeeded(done) { @@ -396,10 +362,10 @@ function install(app, args, progressCallback, callback) { deleteAppDir(app, { removeDirectory: false }, done); // do not remove any symlinked appdata dir }, - function deleteImageIfChanged(done) { - if (!oldManifest || oldManifest.dockerImage === app.manifest.dockerImage) return done(); + async function deleteImageIfChanged() { + if (!oldManifest || oldManifest.dockerImage === app.manifest.dockerImage) return; - docker.deleteImage(oldManifest, done); + await docker.deleteImage(oldManifest); }, // allocating container ip here, lets the users "repair" an app if allocation fails at apps.add time @@ -472,12 +438,12 @@ function install(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error installing app:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } @@ -495,13 +461,13 @@ function backup(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error backing up app:', error); // return to installed state intentionally. the error is stashed only in the task and not the app (the UI shows error state otherwise) - return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null })); } - callback(null); + callback(error); }); } @@ -526,12 +492,12 @@ function create(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error creating :', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } @@ -600,12 +566,12 @@ function changeLocation(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error changing location:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } @@ -639,13 +605,13 @@ function migrateDataDir(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, dataDir: newDataDir }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error migrating data dir:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(); + callback(error); }); } @@ -684,13 +650,13 @@ function configure(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error reconfiguring:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(); + callback(error); }); } @@ -740,10 +706,10 @@ function update(app, args, progressCallback, callback) { // we cannot easily 'recover' from backup failures because we have to revert manfest and portBindings progressCallback.bind(null, { percent: 35, message: 'Cleaning up old install' }), deleteContainers.bind(null, app, { managedOnly: true }), - function deleteImageIfChanged(done) { - if (app.manifest.dockerImage === updateConfig.manifest.dockerImage) return done(); + async function deleteImageIfChanged() { + if (app.manifest.dockerImage === updateConfig.manifest.dockerImage) return; - docker.deleteImage(app.manifest, done); + await docker.deleteImage(app.manifest); }, // only delete unused addons after backup @@ -792,16 +758,16 @@ function update(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, updateTime: new Date() }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error && error.backupError) { debugApp(app, 'update aborted because backup failed', error); - updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })); } else if (error) { debugApp(app, 'Error updating app:', error); - updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); - } else { - callback(null); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } + + callback(error); }); } @@ -827,12 +793,12 @@ function start(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error starting app:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } @@ -854,12 +820,12 @@ function stop(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error starting app:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } @@ -875,12 +841,12 @@ function restart(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 100, message: 'Done' }), updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error starting app:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } @@ -914,12 +880,12 @@ function uninstall(app, args, progressCallback, callback) { progressCallback.bind(null, { percent: 95, message: 'Remove app from database' }), apps.del.bind(null, app.id) - ], function seriesDone(error) { + ], async function seriesDone(error) { if (error) { debugApp(app, 'error uninstalling app:', error); - return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error)); + await safe(updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) })); } - callback(null); + callback(error); }); } diff --git a/src/backuptask.js b/src/backuptask.js index f9c97af70..4c64d4b8d 100644 --- a/src/backuptask.js +++ b/src/backuptask.js @@ -914,16 +914,15 @@ function snapshotApp(app, progressCallback, callback) { const appsBackupConfig = util.callbackify(apps.backupConfig); - appsBackupConfig(app, function (error) { + appsBackupConfig(app, async function (error) { if (error) return callback(error); - services.backupAddons(app, app.manifest.addons, function (error) { - if (error) return callback(error); + [error] = await safe(services.backupAddons(app, app.manifest.addons)); + if (error) return callback(error); - debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`); + debugApp(app, `snapshotApp: took ${(new Date() - startTime)/1000} seconds`); - return callback(null); - }); + return callback(null); }); } diff --git a/src/cron.js b/src/cron.js index c9b22beee..9cf1a739e 100644 --- a/src/cron.js +++ b/src/cron.js @@ -75,7 +75,7 @@ async function startJobs() { gJobs.diskSpaceChecker = new CronJob({ cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration - onTick: () => system.checkDiskSpace(NOOP_CALLBACK), + onTick: async () => await safe(system.checkDiskSpace()), start: true }); @@ -112,7 +112,7 @@ async function startJobs() { gJobs.schedulerSync = new CronJob({ cronTime: constants.TEST ? '*/10 * * * * *' : '00 */1 * * * *', // every minute - onTick: scheduler.sync, + onTick: async () => await safe(scheduler.sync()), start: true }); @@ -124,7 +124,7 @@ async function startJobs() { gJobs.appHealthMonitor = new CronJob({ cronTime: '*/10 * * * * *', // every 10 seconds - onTick: appHealthMonitor.run.bind(null, 10, NOOP_CALLBACK), + onTick: async () => await safe(appHealthMonitor.run(10)), // 10 is the max run time start: true }); diff --git a/src/docker.js b/src/docker.js index f910e3491..2c64a21da 100644 --- a/src/docker.js +++ b/src/docker.js @@ -18,10 +18,8 @@ exports = module.exports = { deleteImage, deleteContainers, createSubcontainer, - getContainerIdByIp, inspect, getContainerIp, - inspectByName: inspect, execContainer, getEvents, memoryUsage, @@ -32,11 +30,11 @@ exports = module.exports = { }; const apps = require('./apps.js'), - async = require('async'), assert = require('assert'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), debug = require('debug')('box:docker'), + delay = require('delay'), Docker = require('dockerode'), path = require('path'), reverseProxy = require('./reverseproxy.js'), @@ -45,7 +43,6 @@ const apps = require('./apps.js'), shell = require('./shell.js'), safe = require('safetydance'), system = require('./system.js'), - util = require('util'), volumes = require('./volumes.js'), _ = require('underscore'); @@ -55,17 +52,13 @@ const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'), const DOCKER_SOCKET_PATH = '/var/run/docker.sock'; const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH }); -function testRegistryConfig(config, callback) { +async function testRegistryConfig(config) { assert.strictEqual(typeof config, 'object'); - assert.strictEqual(typeof callback, 'function'); - if (config.provider === 'noop') return callback(); + if (config.provider === 'noop') return; - gConnection.checkAuth(config, function (error /*, data */) { // this returns a 500 even for auth errors - if (error) return callback(new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' })); - - callback(); - }); + const [error] = await safe(gConnection.checkAuth(config)); // this returns a 500 even for auth errors + if (error) throw new BoxError(BoxError.BAD_FIELD, error, { field: 'serverAddress' }); } function injectPrivateFields(newConfig, currentConfig) { @@ -95,83 +88,77 @@ function ping(callback) { }); } -function getRegistryConfig(image, callback) { +async function getRegistryConfig(image) { // https://github.com/docker/distribution/blob/release/2.7/reference/normalize.go#L62 const parts = image.split('/'); - if (parts.length === 1 || (parts[0].match(/[.:]/) === null)) return callback(null, null); // public docker registry + if (parts.length === 1 || (parts[0].match(/[.:]/) === null)) return null; // public docker registry - util.callbackify(settings.getRegistryConfig)(function (error, registryConfig) { - if (error) return callback(error); + const registryConfig = await settings.getRegistryConfig(); - // https://github.com/apocas/dockerode#pull-from-private-repos - const auth = { - username: registryConfig.username, - password: registryConfig.password, - auth: registryConfig.auth || '', // the auth token at login time - email: registryConfig.email || '', - serveraddress: registryConfig.serverAddress - }; + // https://github.com/apocas/dockerode#pull-from-private-repos + const auth = { + username: registryConfig.username, + password: registryConfig.password, + auth: registryConfig.auth || '', // the auth token at login time + email: registryConfig.email || '', + serveraddress: registryConfig.serverAddress + }; - callback(null, auth); - }); + return auth; } -function pullImage(manifest, callback) { - getRegistryConfig(manifest.dockerImage, function (error, config) { - if (error) return callback(error); +async function pullImage(manifest) { + const config = await getRegistryConfig(manifest.dockerImage); - debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${config ? 'yes' : 'no'}`); + debug(`pullImage: will pull ${manifest.dockerImage}. auth: ${config ? 'yes' : 'no'}`); - gConnection.pull(manifest.dockerImage, { authconfig: config }, function (error, stream) { - if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`)); - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`)); + const [error, stream] = await safe(gConnection.pull(manifest.dockerImage, { authconfig: config })); + if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to pull image ${manifest.dockerImage}. message: ${error.message} statusCode: ${error.statusCode}`); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, `Unable to pull image ${manifest.dockerImage}. Please check the network or if the image needs authentication. statusCode: ${error.statusCode}`); - // https://github.com/dotcloud/docker/issues/1074 says each status message - // is emitted as a chunk - stream.on('data', function (chunk) { - var data = safe.JSON.parse(chunk) || { }; - debug('pullImage: %j', data); + return new Promise((resolve, reject) => { + // https://github.com/dotcloud/docker/issues/1074 says each status message + // is emitted as a chunk + stream.on('data', function (chunk) { + var data = safe.JSON.parse(chunk) || { }; + debug('pullImage: %j', data); - // The data.status here is useless because this is per layer as opposed to per image - if (!data.status && data.error) { - debug('pullImage error %s: %s', manifest.dockerImage, data.errorDetail.message); - } - }); + // The data.status here is useless because this is per layer as opposed to per image + if (!data.status && data.error) { + debug('pullImage error %s: %s', manifest.dockerImage, data.errorDetail.message); + } + }); - stream.on('end', function () { - debug('downloaded image %s', manifest.dockerImage); + stream.on('end', function () { + debug('downloaded image %s', manifest.dockerImage); + resolve(); + }); - callback(null); - }); - - stream.on('error', function (error) { - debug('error pulling image %s: %j', manifest.dockerImage, error); - - callback(new BoxError(BoxError.DOCKER_ERROR, error.message)); - }); + stream.on('error', function (error) { + debug('error pulling image %s: %j', manifest.dockerImage, error); + reject(new BoxError(BoxError.DOCKER_ERROR, error.message)); }); }); } -function downloadImage(manifest, callback) { +async function downloadImage(manifest) { assert.strictEqual(typeof manifest, 'object'); - assert.strictEqual(typeof callback, 'function'); - debug('downloadImage %s', manifest.dockerImage); + debug(`downloadImage ${manifest.dockerImage}`); const image = gConnection.getImage(manifest.dockerImage); - image.inspect(function (error, result) { - if (!error && result) return callback(null); // image is already present locally + const [error, result] = await safe(image.inspect()); + if (!error && result) return; // image is already present locally - let attempt = 1; + for (let times = 0; times < 10; times++) { + debug(`downloadImage: pulling image. attempt ${times+1}`); + const [pullError] = await safe(pullImage(manifest)); + if (pullError && pullError.reason === BoxError.NOT_FOUND) throw pullError; + if (!pullError) break; - async.retry({ times: 10, interval: 5000, errorFilter: e => e.reason !== BoxError.NOT_FOUND }, function (retryCallback) { - debug('Downloading image %s. attempt: %s', manifest.dockerImage, attempt++); - - pullImage(manifest, retryCallback); - }, callback); - }); + await delay(5000); + } } async function getVolumeMounts(app) { @@ -200,16 +187,15 @@ async function getVolumeMounts(app) { return mounts; } -function getAddonMounts(app, callback) { +async function getAddonMounts(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); let mounts = []; const addons = app.manifest.addons; - if (!addons) return callback(null, mounts); + if (!addons) return mounts; - async.eachSeries(Object.keys(addons), async function (addon) { + for (const addon of Object.keys(addons)) { switch (addon) { case 'localstorage': mounts.push({ @@ -219,7 +205,7 @@ function getAddonMounts(app, callback) { ReadOnly: false }); - return; + break; case 'tls': { const bundle = await reverseProxy.getCertificatePath(app.fqdn, app.domain); @@ -237,28 +223,22 @@ function getAddonMounts(app, callback) { ReadOnly: true }); - return; + break; } default: - return; + break; } - }, function (error) { - callback(error, mounts); - }); + } + + return mounts; } -async function getMounts(app, callback) { +async function getMounts(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); - const [error, volumeMounts] = await safe(getVolumeMounts(app)); - if (error) return callback(error); - - getAddonMounts(app, function (error, addonMounts) { - if (error) return callback(error); - - callback(null, volumeMounts.concat(addonMounts)); - }); + const volumeMounts = await getVolumeMounts(app); + const addonMounts = await getAddonMounts(app); + return volumeMounts.concat(addonMounts); } function getAddresses() { @@ -278,22 +258,20 @@ function getAddresses() { return addresses; } -function createSubcontainer(app, name, cmd, options, callback) { +async function createSubcontainer(app, name, cmd, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof name, 'string'); assert(!cmd || Array.isArray(cmd)); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); let isAppContainer = !cmd; // non app-containers are like scheduler - var manifest = app.manifest; - var exposedPorts = {}, dockerPortBindings = { }; - var domain = app.fqdn; - + const manifest = app.manifest; + const exposedPorts = {}, dockerPortBindings = { }; + const domain = app.fqdn; const envPrefix = manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - let stdEnv = [ + const stdEnv = [ 'CLOUDRON=1', 'CLOUDRON_PROXY_IP=172.18.0.1', `CLOUDRON_APP_HOSTNAME=${app.id}`, @@ -303,13 +281,13 @@ function createSubcontainer(app, name, cmd, options, callback) { `${envPrefix}APP_DOMAIN=${domain}` ]; - var portEnv = []; - for (let portName in app.portBindings) { + const portEnv = []; + for (const portName in app.portBindings) { const hostPort = app.portBindings[portName]; const portType = (manifest.tcpPorts && portName in manifest.tcpPorts) ? 'tcp' : 'udp'; const ports = portType == 'tcp' ? manifest.tcpPorts : manifest.udpPorts; - var containerPort = ports[portName].containerPort || hostPort; + const containerPort = ports[portName].containerPort || hostPort; // docker portBindings requires ports to be exposed exposedPorts[`${containerPort}/${portType}`] = {}; @@ -319,7 +297,7 @@ function createSubcontainer(app, name, cmd, options, callback) { dockerPortBindings[`${containerPort}/${portType}`] = hostIps.map(hip => { return { HostIp: hip, HostPort: hostPort + '' }; }); } - let appEnv = []; + const appEnv = []; Object.keys(app.env).forEach(function (name) { appEnv.push(`${name}=${app.env[name]}`); }); let memoryLimit = apps.getMemoryLimit(app); @@ -328,219 +306,194 @@ function createSubcontainer(app, name, cmd, options, callback) { // if required, we can make this a manifest and runtime argument later if (!isAppContainer) memoryLimit *= 2; - getMounts(app, async function (error, mounts) { - if (error) return callback(error); + const mounts = await getMounts(app); - const [getEnvError, addonEnv] = await safe(services.getEnvironment(app)); - if (getEnvError) return callback(getEnvError); + const addonEnv = await services.getEnvironment(app); - let containerOptions = { - name: name, // for referencing containers - Tty: isAppContainer, - Image: app.manifest.dockerImage, - Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd, - Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv), - ExposedPorts: isAppContainer ? exposedPorts : { }, - Volumes: { // see also ReadonlyRootfs - '/tmp': {}, - '/run': {} + let containerOptions = { + name: name, // for referencing containers + Tty: isAppContainer, + Image: app.manifest.dockerImage, + Cmd: (isAppContainer && app.debugMode && app.debugMode.cmd) ? app.debugMode.cmd : cmd, + Env: stdEnv.concat(addonEnv).concat(portEnv).concat(appEnv), + ExposedPorts: isAppContainer ? exposedPorts : { }, + Volumes: { // see also ReadonlyRootfs + '/tmp': {}, + '/run': {} + }, + Labels: { + 'fqdn': app.fqdn, + 'appId': app.id, + 'isSubcontainer': String(!isAppContainer), + 'isCloudronManaged': String(true) + }, + HostConfig: { + Mounts: mounts, + LogConfig: { + Type: 'syslog', + Config: { + 'tag': app.id, + 'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings() + 'syslog-format': 'rfc5424' + } }, - Labels: { - 'fqdn': app.fqdn, - 'appId': app.id, - 'isSubcontainer': String(!isAppContainer), - 'isCloudronManaged': String(true) + Memory: system.getMemoryAllocation(memoryLimit), + MemorySwap: memoryLimit, // Memory + Swap + PortBindings: isAppContainer ? dockerPortBindings : { }, + PublishAllPorts: false, + ReadonlyRootfs: app.debugMode ? !!app.debugMode.readonlyRootfs : true, + RestartPolicy: { + 'Name': isAppContainer ? 'unless-stopped' : 'no', + 'MaximumRetryCount': 0 }, - HostConfig: { - Mounts: mounts, - LogConfig: { - Type: 'syslog', - Config: { - 'tag': app.id, - 'syslog-address': 'udp://127.0.0.1:2514', // see apps.js:validatePortBindings() - 'syslog-format': 'rfc5424' - } - }, - Memory: system.getMemoryAllocation(memoryLimit), - MemorySwap: memoryLimit, // Memory + Swap - PortBindings: isAppContainer ? dockerPortBindings : { }, - PublishAllPorts: false, - ReadonlyRootfs: app.debugMode ? !!app.debugMode.readonlyRootfs : true, - RestartPolicy: { - 'Name': isAppContainer ? 'unless-stopped' : 'no', - 'MaximumRetryCount': 0 - }, - CpuShares: app.cpuShares, - VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ], - SecurityOpt: [ 'apparmor=docker-cloudron-app' ], - CapAdd: [], - CapDrop: [] + CpuShares: app.cpuShares, + VolumesFrom: isAppContainer ? null : [ app.containerId + ':rw' ], + SecurityOpt: [ 'apparmor=docker-cloudron-app' ], + CapAdd: [], + CapDrop: [] + } + }; + + // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail + // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns + // name to look up the internal docker ip. this makes curl from within container fail + // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns. + // Hostname cannot be set with container NetworkMode. Subcontainers run is the network space of the app container + // This is done to prevent lots of up/down events and iptables locking + if (isAppContainer) { + containerOptions.Hostname = app.id; + containerOptions.HostConfig.NetworkMode = 'cloudron'; // user defined bridge network + containerOptions.HostConfig.Dns = ['172.18.0.1']; // use internal dns + containerOptions.HostConfig.DnsSearch = ['.']; // use internal dns + + containerOptions.NetworkingConfig = { + EndpointsConfig: { + cloudron: { + IPAMConfig: { + IPv4Address: app.containerIp + }, + Aliases: [ name ] // adds hostname entry with container name + } } }; + } else { + containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`; // scheduler containers must have same IP as app for various addon auth + } - // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail - // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns - // name to look up the internal docker ip. this makes curl from within container fail - // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns. - // Hostname cannot be set with container NetworkMode. Subcontainers run is the network space of the app container - // This is done to prevent lots of up/down events and iptables locking - if (isAppContainer) { - containerOptions.Hostname = app.id; - containerOptions.HostConfig.NetworkMode = 'cloudron'; // user defined bridge network - containerOptions.HostConfig.Dns = ['172.18.0.1']; // use internal dns - containerOptions.HostConfig.DnsSearch = ['.']; // use internal dns + const capabilities = manifest.capabilities || []; - containerOptions.NetworkingConfig = { - EndpointsConfig: { - cloudron: { - IPAMConfig: { - IPv4Address: app.containerIp - }, - Aliases: [ name ] // adds hostname entry with container name - } - } - }; - } else { - containerOptions.HostConfig.NetworkMode = `container:${app.containerId}`; // scheduler containers must have same IP as app for various addon auth - } + // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities + if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW'); + if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping + if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker - var capabilities = manifest.capabilities || []; + if (capabilities.includes('vaapi') && safe.fs.existsSync('/dev/dri')) { + containerOptions.HostConfig.Devices = [ + { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' } + ]; + } - // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities - if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW'); - if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping - if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker + containerOptions = _.extend(containerOptions, options); - if (capabilities.includes('vaapi') && safe.fs.existsSync('/dev/dri')) { - containerOptions.HostConfig.Devices = [ - { PathOnHost: '/dev/dri', PathInContainer: '/dev/dri', CgroupPermissions: 'rwm' } - ]; - } + const [createError, container] = await safe(gConnection.createContainer(containerOptions)); + if (createError && createError.statusCode === 409) throw new BoxError(BoxError.ALREADY_EXISTS, createError); + if (createError) throw new BoxError(BoxError.DOCKER_ERROR, createError); - containerOptions = _.extend(containerOptions, options); - - gConnection.createContainer(containerOptions, function (error, container) { - if (error && error.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error)); - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - - callback(null, container); - }); - }); + return container; } -function createContainer(app, callback) { - createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */, callback); +async function createContainer(app) { + return await createSubcontainer(app, app.id /* name */, null /* cmd */, { } /* options */); } -function startContainer(containerId, callback) { +async function startContainer(containerId) { assert.strictEqual(typeof containerId, 'string'); - assert.strictEqual(typeof callback, 'function'); - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - container.start(function (error) { - if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); - if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable - if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started - - return callback(null); - }); + const [error] = await safe(container.start()); + if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND); + if (error && error.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, error); // e.g start.sh is not executable + if (error && error.statusCode !== 304) throw new BoxError(BoxError.DOCKER_ERROR, error); // 304 means already started } -function restartContainer(containerId, callback) { +async function restartContainer(containerId) { assert.strictEqual(typeof containerId, 'string'); - assert.strictEqual(typeof callback, 'function'); - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - container.restart(function (error) { - if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); - if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable - if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - - return callback(null); - }); + const [error] = await safe(container.restart()); + if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND); + if (error && error.statusCode === 400) throw new BoxError(BoxError.BAD_FIELD, error); // e.g start.sh is not executable + if (error && error.statusCode !== 204) throw new BoxError(BoxError.DOCKER_ERROR, error); } -function stopContainer(containerId, callback) { +async function stopContainer(containerId) { assert(!containerId || typeof containerId === 'string'); - assert.strictEqual(typeof callback, 'function'); if (!containerId) { debug('No previous container to stop'); - return callback(); + return; } - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - var options = { + const options = { t: 10 // wait for 10 seconds before killing it }; - container.stop(options, function (error) { - if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message)); + let [error] = await safe(container.stop(options)); + if (error && (error.statusCode !== 304 && error.statusCode !== 404)) throw new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message); - container.wait(function (error/*, data */) { - if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message)); - - return callback(null); - }); - }); + [error] = await safe(container.wait()); + if (error && (error.statusCode !== 304 && error.statusCode !== 404)) throw new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message); } -function deleteContainer(containerId, callback) { // id can also be name +async function deleteContainer(containerId) { // id can also be name assert(!containerId || typeof containerId === 'string'); - assert.strictEqual(typeof callback, 'function'); - if (containerId === null) return callback(null); + if (containerId === null) return null; - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - var removeOptions = { + const removeOptions = { force: true, // kill container if it's running v: true // removes volumes associated with the container (but not host mounts) }; - container.remove(removeOptions, function (error) { - if (error && error.statusCode === 404) return callback(null); + const [error] = await safe(container.remove(removeOptions)); + if (error && error.statusCode === 404) return; - if (error) { - debug('Error removing container %s : %j', containerId, error); - return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - } - - callback(null); - }); + if (error) { + debug('Error removing container %s : %j', containerId, error); + throw new BoxError(BoxError.DOCKER_ERROR, error); + } } -function deleteContainers(appId, options, callback) { +async function deleteContainers(appId, options) { assert.strictEqual(typeof appId, 'string'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - let labels = [ 'appId=' + appId ]; + const labels = [ 'appId=' + appId ]; if (options.managedOnly) labels.push('isCloudronManaged=true'); - gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) }, function (error, containers) { - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + const [error, containers] = await safe(gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: labels }) })); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); - async.eachSeries(containers, function (container, iteratorDone) { - deleteContainer(container.Id, iteratorDone); - }, callback); - }); + for (const container of containers) { + await deleteContainer(container.Id); + } } -function stopContainers(appId, callback) { +async function stopContainers(appId) { assert.strictEqual(typeof appId, 'string'); - assert.strictEqual(typeof callback, 'function'); - gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) { - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + const [error, containers] = await safe(gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) })); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); - async.eachSeries(containers, function (container, iteratorDone) { - stopContainer(container.Id, iteratorDone); - }, callback); - }); + for (const container of containers) { + await stopContainer(container.Id); + } } function deleteImage(manifest, callback) { @@ -573,114 +526,79 @@ function deleteImage(manifest, callback) { }); } -function getContainerIdByIp(ip, callback) { - assert.strictEqual(typeof ip, 'string'); - assert.strictEqual(typeof callback, 'function'); - - gConnection.getNetwork('cloudron').inspect(function (error, bridge) { - if (error && error.statusCode === 404) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Unable to find the cloudron network')); - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - - var containerId; - for (var id in bridge.Containers) { - if (bridge.Containers[id].IPv4Address.indexOf(ip + '/16') === 0) { - containerId = id; - break; - } - } - if (!containerId) return callback(new BoxError(BoxError.DOCKER_ERROR, 'No container with that ip')); - - callback(null, containerId); - }); -} - -function inspect(containerId, callback) { +async function inspect(containerId) { assert.strictEqual(typeof containerId, 'string'); - assert.strictEqual(typeof callback, 'function'); - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - container.inspect(function (error, result) { - if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to find container ${containerId}`)); - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + const [error, result] = await safe(container.inspect()); + if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND, `Unable to find container ${containerId}`); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); - callback(null, result); - }); + return result; } -function getContainerIp(containerId, callback) { +async function getContainerIp(containerId) { assert.strictEqual(typeof containerId, 'string'); - assert.strictEqual(typeof callback, 'function'); - if (constants.TEST) return callback(null, '127.0.5.5'); + if (constants.TEST) return '127.0.5.5'; - inspect(containerId, function (error, result) { - if (error) return callback(error); + const result = await inspect(containerId); - const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null); - if (!ip) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error getting container IP')); + const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null); + if (!ip) throw new BoxError(BoxError.DOCKER_ERROR, 'Error getting container IP'); - callback(null, ip); - }); + return ip; } -function execContainer(containerId, options, callback) { +async function execContainer(containerId, options) { assert.strictEqual(typeof containerId, 'string'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - container.exec(options.execOptions, function (error, exec) { - if (error && error.statusCode === 409) return callback(new BoxError(BoxError.BAD_STATE, error.message)); // container restarting/not running - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + const [error, exec] = await safe(container.exec(options.execOptions)); + if (error && error.statusCode === 409) throw new BoxError(BoxError.BAD_STATE, error.message); // container restarting/not running + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); - exec.start(options.startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) { - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + const [startError, stream] = await safe(exec.start(options.startOptions)); /* in hijacked mode, stream is a net.socket */ + if (startError) throw new BoxError(BoxError.DOCKER_ERROR, startError); - if (options.rows && options.columns) { - // there is a race where resizing too early results in a 404 "no such exec" - // https://git.cloudron.io/cloudron/box/issues/549 - setTimeout(function () { - exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); }); - }, 2000); - } + if (options.rows && options.columns) { + // there is a race where resizing too early results in a 404 "no such exec" + // https://git.cloudron.io/cloudron/box/issues/549 + setTimeout(function () { + exec.resize({ h: options.rows, w: options.columns }, function (error) { if (error) debug('Error resizing console', error); }); + }, 2000); + } - callback(null, stream); - }); - }); + return stream; } -function getEvents(options, callback) { +async function getEvents(options) { assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - gConnection.getEvents(options, function (error, stream) { - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - - callback(null, stream); - }); + const [error, stream] = await safe(gConnection.getEvents(options)); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); + return stream; } -function memoryUsage(containerId, callback) { +async function memoryUsage(containerId) { assert.strictEqual(typeof containerId, 'string'); - assert.strictEqual(typeof callback, 'function'); - var container = gConnection.getContainer(containerId); + const container = gConnection.getContainer(containerId); - container.stats({ stream: false }, function (error, result) { - if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND)); - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + const [error, result] = await safe(container.stats({ stream: false })); + if (error && error.statusCode === 404) throw new BoxError(BoxError.NOT_FOUND); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); - callback(null, result); - }); + return result; } -function createVolume(name, volumeDataDir, labels, callback) { +async function createVolume(name, volumeDataDir, labels) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof volumeDataDir, 'string'); assert.strictEqual(typeof labels, 'object'); - assert.strictEqual(typeof callback, 'function'); const volumeOptions = { Name: name, @@ -694,69 +612,56 @@ function createVolume(name, volumeDataDir, labels, callback) { }; // requires sudo because the path can be outside appsdata - shell.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {}, function (error) { - if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`)); + let [error] = await safe(shell.promises.sudo('createVolume', [ MKDIRVOLUME_CMD, volumeDataDir ], {})); + if (error) throw new BoxError(BoxError.FS_ERROR, `Error creating app data dir: ${error.message}`); - gConnection.createVolume(volumeOptions, function (error) { - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); - - callback(); - }); - }); + [error] = await safe(gConnection.createVolume(volumeOptions)); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); } -function clearVolume(name, options, callback) { +async function clearVolume(name, options) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); let volume = gConnection.getVolume(name); - volume.inspect(function (error, v) { - if (error && error.statusCode === 404) return callback(); - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); + let [error, v] = await safe(volume.inspect()); + if (error && error.statusCode === 404) return; + if (error) throw new BoxError(BoxError.DOCKER_ERROR, error); - const volumeDataDir = v.Options.device; - shell.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {}, function (error) { - if (error) return callback(new BoxError(BoxError.FS_ERROR, error)); - - callback(); - }); - }); + const volumeDataDir = v.Options.device; + [error] = await shell.promises.sudo('clearVolume', [ CLEARVOLUME_CMD, options.removeDirectory ? 'rmdir' : 'clear', volumeDataDir ], {}); + if (error) throw new BoxError(BoxError.FS_ERROR, error); } // this only removes the volume and not the data -function removeVolume(name, callback) { +async function removeVolume(name) { assert.strictEqual(typeof name, 'string'); - assert.strictEqual(typeof callback, 'function'); let volume = gConnection.getVolume(name); - volume.remove(function (error) { - if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`)); - - callback(); - }); + const [error] = await safe(volume.remove()); + if (error && error.statusCode !== 404) throw new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`); } -function info(callback) { - assert.strictEqual(typeof callback, 'function'); - - gConnection.info(function (error, result) { - if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker')); - - callback(null, result); - }); +async function info() { + const [error, result] = await safe(gConnection.info()); + if (error) throw new BoxError(BoxError.DOCKER_ERROR, 'Error connecting to docker'); + return result; } -function update(name, memory, memorySwap, callback) { +async function update(name, memory, memorySwap) { assert.strictEqual(typeof name, 'string'); assert.strictEqual(typeof memory, 'number'); assert.strictEqual(typeof memorySwap, 'number'); - assert.strictEqual(typeof callback, 'function'); const args = `update --memory ${memory} --memory-swap ${memorySwap} ${name}`.split(' '); // scale back db containers, if possible. this is retried because updating memory constraints can fail // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy - async.retry({ times: 10, interval: 60 * 1000 }, function (retryCallback) { - shell.spawn(`update(${name})`, '/usr/bin/docker', args, { }, retryCallback); - }, callback); + + for (let times = 0; times < 10; times++) { + const [error] = await safe(shell.promises.spawn(`update(${name})`, '/usr/bin/docker', args, { })); + if (!error) return; + await delay(60 * 1000); + } + + throw new BoxError(BoxError.DOCKER_ERROR, 'Unable to update container'); } diff --git a/src/domains.js b/src/domains.js index c34b56c83..f5e221e5a 100644 --- a/src/domains.js +++ b/src/domains.js @@ -285,7 +285,7 @@ async function del(domain, auditSource) { eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain }); - mail.onDomainRemoved(domain, NOOP_CALLBACK); + safe(mail.onDomainRemoved(domain)); } async function clear() { diff --git a/src/ldap.js b/src/ldap.js index 53da2f780..131d8128d 100644 --- a/src/ldap.js +++ b/src/ldap.js @@ -18,7 +18,7 @@ const addonConfigs = require('./addonconfigs.js'), ldap = require('ldapjs'), mail = require('./mail.js'), safe = require('safetydance'), - services = require('./services.js'), + settings = require('./settings.js'), users = require('./users.js'); var gServer = null; @@ -550,14 +550,13 @@ async function authenticateSftp(req, res, next) { res.end(); } -function loadSftpConfig(req, res, next) { - services.getServiceConfig('sftp', function (error, serviceConfig) { - if (error) return next(new ldap.OperationsError(error.toString())); +async function loadSftpConfig(req, res, next) { + const [error, servicesConfig] = await settings.getServicesConfig(); + if (error) return next(new ldap.OperationsError(error.toString())); - req.requireAdmin = serviceConfig.requireAdmin; - - next(); - }); + const sftpConfig = servicesConfig['sftp'] || {}; + req.requireAdmin = sftpConfig.requireAdmin; + next(); } async function userSearchSftp(req, res, next) { diff --git a/src/mail.js b/src/mail.js index 2439f2b0f..d938b7714 100644 --- a/src/mail.js +++ b/src/mail.js @@ -87,13 +87,13 @@ const assert = require('assert'), nodemailer = require('nodemailer'), path = require('path'), paths = require('./paths.js'), - request = require('request'), reverseProxy = require('./reverseproxy.js'), safe = require('safetydance'), services = require('./services.js'), settings = require('./settings.js'), shell = require('./shell.js'), smtpTransport = require('nodemailer-smtp-transport'), + superagent = require('superagent'), sysinfo = require('./sysinfo.js'), system = require('./system.js'), tasks = require('./tasks.js'), @@ -716,9 +716,7 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) { } async function getMailAuth() { - const dockerInspect = util.promisify(docker.inspect); - - const data = await dockerInspect('mail'); + const data = await docker.inspect('mail'); const ip = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress'); if (!ip) throw new BoxError(BoxError.MAIL_ERROR, 'Error querying mail server IP'); @@ -737,18 +735,14 @@ async function getMailAuth() { }; } -function restartMail(callback) { - assert.strictEqual(typeof callback, 'function'); +async function restartMail() { + if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return; - if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback(); + const servicesConfig = await settings.getServicesConfig(); + const mailConfig = servicesConfig['mail'] || {}; - services.getServiceConfig('mail', async function (error, serviceConfig) { - if (error) return callback(error); - - debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} dashboardDomain:${settings.dashboardDomain()}`); - [error] = await safe(configureMail(settings.mailFqdn(), settings.dashboardDomain(), serviceConfig)); - callback(error); - }); + debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} dashboardDomain:${settings.dashboardDomain()}`); + await configureMail(settings.mailFqdn(), settings.dashboardDomain(), mailConfig); } async function restartMailIfActivated() { @@ -759,7 +753,7 @@ async function restartMailIfActivated() { return; // not provisioned yet, do not restart container after dns setup } - await util.promisify(restartMail)(); + await restartMail(); } async function handleCertChanged() { @@ -1034,11 +1028,10 @@ function onDomainAdded(domain, callback) { ], callback); } -function onDomainRemoved(domain, callback) { +async function onDomainRemoved(domain) { assert.strictEqual(typeof domain, 'string'); - assert.strictEqual(typeof callback, 'function'); - restartMail(callback); + await restartMail(); } async function clearDomains() { @@ -1061,7 +1054,7 @@ async function setMailFromValidation(domain, enabled) { await updateDomain(domain, { mailFromValidation: enabled }); - restartMail(NOOP_CALLBACK); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) + safe(restartMail()); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) } async function setBanner(domain, banner) { @@ -1070,7 +1063,7 @@ async function setBanner(domain, banner) { await updateDomain(domain, { banner }); - restartMail(NOOP_CALLBACK); + safe(restartMail()); } async function setCatchAllAddress(domain, addresses) { @@ -1079,7 +1072,7 @@ async function setCatchAllAddress(domain, addresses) { await updateDomain(domain, { catchAll: addresses }); - restartMail(NOOP_CALLBACK); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) + safe(restartMail()); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini) } async function setMailRelay(domain, relay, options) { @@ -1100,7 +1093,7 @@ async function setMailRelay(domain, relay, options) { await updateDomain(domain, { relay: relay }); - restartMail(NOOP_CALLBACK); + safe(restartMail()); } async function setMailEnabled(domain, enabled, auditSource) { @@ -1110,7 +1103,7 @@ async function setMailEnabled(domain, enabled, auditSource) { await updateDomain(domain, { enabled: enabled }); - restartMail(NOOP_CALLBACK); + safe(restartMail()); await eventlog.add(enabled ? eventlog.ACTION_MAIL_ENABLED : eventlog.ACTION_MAIL_DISABLED, auditSource, { domain }); } @@ -1252,21 +1245,20 @@ async function updateMailbox(name, domain, data, auditSource) { eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active }); } -function removeSolrIndex(mailbox, callback) { +async function removeSolrIndex(mailbox) { assert.strictEqual(typeof mailbox, 'string'); - assert.strictEqual(typeof callback, 'function'); - services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) { - if (error) return callback(error); + const addonDetails = await services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN'); - request.post(`https://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`, { timeout: 2000, rejectUnauthorized: false, json: { mailbox } }, function (error, response) { - if (error) return callback(error); + const [error, response] = await safe(superagent.post(`https://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`) + .timeout(2000) + .disableTLSCerts() + .send({ mailbox }) + .ok(() => true)); - if (response.statusCode !== 200) return callback(new Error(`Error removing solr index - ${response.statusCode} ${JSON.stringify(response.body)}`)); + if (error) throw new BoxError(BoxError.MAIL_ERROR, `Could not remove solr index: ${error.message}`); - callback(null); - }); - }); + if (response.status !== 200) throw new BoxError(BoxError.MAIL_ERROR, `Error removing solr index - ${response.status} ${JSON.stringify(response.body)}`); } async function delMailbox(name, domain, options, auditSource) { @@ -1285,7 +1277,9 @@ async function delMailbox(name, domain, options, auditSource) { const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]); if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'); - removeSolrIndex(mailbox, NOOP_CALLBACK); + const [error] = await safe(removeSolrIndex(mailbox)); + if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`); + eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain }); } diff --git a/src/routes/apps.js b/src/routes/apps.js index 1002b9a8a..4d8da6624 100644 --- a/src/routes/apps.js +++ b/src/routes/apps.js @@ -626,92 +626,90 @@ function demuxStream(stream, stdin) { }); } -function exec(req, res, next) { +async function exec(req, res, next) { assert.strictEqual(typeof req.resource, 'object'); - var cmd = null; + let cmd = null; if (req.query.cmd) { cmd = safe.JSON.parse(req.query.cmd); if (!Array.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1')); } - var columns = req.query.columns ? parseInt(req.query.columns, 10) : null; + const columns = req.query.columns ? parseInt(req.query.columns, 10) : null; if (isNaN(columns)) return next(new HttpError(400, 'columns must be a number')); - var rows = req.query.rows ? parseInt(req.query.rows, 10) : null; + const rows = req.query.rows ? parseInt(req.query.rows, 10) : null; if (isNaN(rows)) return next(new HttpError(400, 'rows must be a number')); - var tty = req.query.tty === 'true'; + const tty = req.query.tty === 'true'; if (safe.query(req.resource, 'manifest.addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is requied to exec app with docker addon')); // in a badly configured reverse proxy, we might be here without an upgrade if (req.headers['upgrade'] !== 'tcp') return next(new HttpError(404, 'exec requires TCP upgrade')); - apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) { - if (error) return next(BoxError.toHttpError(error)); + const [error, duplexStream] = await safe(apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty })); + if (error) return next(BoxError.toHttpError(error)); - req.clearTimeout(); - res.sendUpgradeHandshake(); + req.clearTimeout(); + res.sendUpgradeHandshake(); - // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged. - duplexStream.pipe(res.socket); + // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged. + duplexStream.pipe(res.socket); - if (tty) { - res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit - } else { - demuxStream(res.socket, duplexStream); - res.socket.on('error', function () { duplexStream.end(); }); - res.socket.on('end', function () { duplexStream.end(); }); - } - }); + if (tty) { + res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit + } else { + demuxStream(res.socket, duplexStream); + res.socket.on('error', function () { duplexStream.end(); }); + res.socket.on('end', function () { duplexStream.end(); }); + } } -function execWebSocket(req, res, next) { +async function execWebSocket(req, res, next) { assert.strictEqual(typeof req.resource, 'object'); - var cmd = null; + let cmd = null; if (req.query.cmd) { cmd = safe.JSON.parse(req.query.cmd); if (!Array.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1')); } - var columns = req.query.columns ? parseInt(req.query.columns, 10) : null; + const columns = req.query.columns ? parseInt(req.query.columns, 10) : null; if (isNaN(columns)) return next(new HttpError(400, 'columns must be a number')); - var rows = req.query.rows ? parseInt(req.query.rows, 10) : null; + const rows = req.query.rows ? parseInt(req.query.rows, 10) : null; if (isNaN(rows)) return next(new HttpError(400, 'rows must be a number')); - var tty = req.query.tty === 'true' ? true : false; + const tty = req.query.tty === 'true' ? true : false; // in a badly configured reverse proxy, we might be here without an upgrade if (req.headers['upgrade'] !== 'websocket') return next(new HttpError(404, 'exec requires websocket')); - apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) { - if (error) return next(BoxError.toHttpError(error)); + const [error, duplexStream] = await safe(apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty })); + if (error) return next(BoxError.toHttpError(error)); - req.clearTimeout(); + req.clearTimeout(); - res.handleUpgrade(function (ws) { - duplexStream.on('end', function () { ws.close(); }); - duplexStream.on('close', function () { ws.close(); }); - duplexStream.on('error', function (error) { - debug('duplexStream error:', error); - }); - duplexStream.on('data', function (data) { - if (ws.readyState !== WebSocket.OPEN) return; - ws.send(data.toString()); - }); + res.handleUpgrade(function (ws) { + duplexStream.on('end', function () { ws.close(); }); + duplexStream.on('close', function () { ws.close(); }); + duplexStream.on('error', function (error) { + debug('duplexStream error:', error); + }); + duplexStream.on('data', function (data) { + if (ws.readyState !== WebSocket.OPEN) return; + ws.send(data.toString()); + }); - ws.on('error', function (error) { - debug('websocket error:', error); - }); - ws.on('message', function (msg) { - duplexStream.write(msg); - }); - ws.on('close', function () { - // Clean things up, if any? - }); + ws.on('error', function (error) { + debug('websocket error:', error); + }); + ws.on('message', function (msg) { + duplexStream.write(msg); + }); + ws.on('close', function () { + // Clean things up, if any? }); }); } diff --git a/src/routes/cloudron.js b/src/routes/cloudron.js index 215454de7..e33005198 100644 --- a/src/routes/cloudron.js +++ b/src/routes/cloudron.js @@ -163,20 +163,18 @@ async function getConfig(req, res, next) { next(new HttpSuccess(200, cloudronConfig)); } -function getDisks(req, res, next) { - system.getDisks(function (error, result) { - if (error) return next(BoxError.toHttpError(error)); +async function getDisks(req, res, next) { + const [error, result] = await safe(system.getDisks()); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(200, result)); - }); + next(new HttpSuccess(200, result)); } -function getMemory(req, res, next) { - system.getMemory(function (error, result) { - if (error) return next(BoxError.toHttpError(error)); +async function getMemory(req, res, next) { + const [error, result] = await safe(system.getMemory()); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(200, result)); - }); + next(new HttpSuccess(200, result)); } function update(req, res, next) { diff --git a/src/routes/filemanager.js b/src/routes/filemanager.js index 1ab2e384c..f700e15eb 100644 --- a/src/routes/filemanager.js +++ b/src/routes/filemanager.js @@ -8,35 +8,35 @@ const assert = require('assert'), BoxError = require('../boxerror.js'), middleware = require('../middleware/index.js'), HttpError = require('connect-lastmile').HttpError, + safe = require('safetydance'), services = require('../services.js'), url = require('url'); -function proxy(req, res, next) { +async function proxy(req, res, next) { assert.strictEqual(typeof req.params.id, 'string'); const id = req.params.id; // app id or volume id req.clearTimeout(); - services.getContainerDetails('sftp', 'CLOUDRON_SFTP_TOKEN', function (error, result) { - if (error) return next(BoxError.toHttpError(error)); + const [error, result] = await safe(services.getContainerDetails('sftp', 'CLOUDRON_SFTP_TOKEN')); + if (error) return next(BoxError.toHttpError(error)); - let parsedUrl = url.parse(req.url, true /* parseQueryString */); - parsedUrl.query['access_token'] = result.token; + let parsedUrl = url.parse(req.url, true /* parseQueryString */); + parsedUrl.query['access_token'] = result.token; - req.url = url.format({ pathname: `/files/${id}/${encodeURIComponent(req.params[0])}`, query: parsedUrl.query }); // params[0] already contains leading '/' + req.url = url.format({ pathname: `/files/${id}/${encodeURIComponent(req.params[0])}`, query: parsedUrl.query }); // params[0] already contains leading '/' - const proxyOptions = url.parse(`https://${result.ip}:3000`); - proxyOptions.rejectUnauthorized = false; - const fileManagerProxy = middleware.proxy(proxyOptions); + const proxyOptions = url.parse(`https://${result.ip}:3000`); + proxyOptions.rejectUnauthorized = false; + const fileManagerProxy = middleware.proxy(proxyOptions); - fileManagerProxy(req, res, function (error) { - if (!error) return next(); + fileManagerProxy(req, res, function (error) { + if (!error) return next(); - if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to filemanager server')); - if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query filemanager server')); + if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to filemanager server')); + if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query filemanager server')); - next(new HttpError(500, error)); - }); + next(new HttpError(500, error)); }); } diff --git a/src/routes/mailserver.js b/src/routes/mailserver.js index d8c7c62bb..eea995710 100644 --- a/src/routes/mailserver.js +++ b/src/routes/mailserver.js @@ -25,7 +25,7 @@ function restart(req, res, next) { next(); } -function proxy(req, res, next) { +async function proxy(req, res, next) { let parsedUrl = url.parse(req.url, true /* parseQueryString */); const pathname = req.path.split('/').pop(); @@ -34,25 +34,24 @@ function proxy(req, res, next) { delete req.headers['authorization']; delete req.headers['cookies']; - services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) { - if (error) return next(BoxError.toHttpError(error)); + const [error, addonDetails] = await safe(services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN')); + if (error) return next(BoxError.toHttpError(error)); - parsedUrl.query['access_token'] = addonDetails.token; - req.url = url.format({ pathname: pathname, query: parsedUrl.query }); + parsedUrl.query['access_token'] = addonDetails.token; + req.url = url.format({ pathname: pathname, query: parsedUrl.query }); - const proxyOptions = url.parse(`https://${addonDetails.ip}:3000`); - proxyOptions.rejectUnauthorized = false; - const mailserverProxy = middleware.proxy(proxyOptions); + const proxyOptions = url.parse(`https://${addonDetails.ip}:3000`); + proxyOptions.rejectUnauthorized = false; + const mailserverProxy = middleware.proxy(proxyOptions); - req.clearTimeout(); // TODO: add timeout to mail server proxy logic instead of this - mailserverProxy(req, res, function (error) { - if (!error) return next(); + req.clearTimeout(); // TODO: add timeout to mail server proxy logic instead of this + mailserverProxy(req, res, function (error) { + if (!error) return next(); - if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to mail server')); - if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query mail server')); + if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to mail server')); + if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query mail server')); - next(new HttpError(500, error)); - }); + next(new HttpError(500, error)); }); } diff --git a/src/routes/provision.js b/src/routes/provision.js index 98d9fa669..8f2644757 100644 --- a/src/routes/provision.js +++ b/src/routes/provision.js @@ -16,9 +16,9 @@ const assert = require('assert'), HttpSuccess = require('connect-lastmile').HttpSuccess, paths = require('../paths.js'), provision = require('../provision.js'), - request = require('request'), safe = require('safetydance'), - settings = require('../settings.js'); + settings = require('../settings.js'), + superagent = require('superagent'); function setupTokenAuth(req, res, next) { assert.strictEqual(typeof req.body, 'object'); @@ -32,20 +32,19 @@ function setupTokenAuth(req, res, next) { return next(); } -function providerTokenAuth(req, res, next) { +async function providerTokenAuth(req, res, next) { assert.strictEqual(typeof req.body, 'object'); if (settings.provider() === 'ami') { if (typeof req.body.providerToken !== 'string' || !req.body.providerToken) return next(new HttpError(400, 'providerToken must be a non empty string')); - request.get('http://169.254.169.254/latest/meta-data/instance-id', { timeout: 30 * 1000 }, function (error, result) { - if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`)); - if (result.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${result.statusCode}`)); + const [error, response] = await superagent.get('http://169.254.169.254/latest/meta-data/instance-id').timeout(30 * 1000).ok(() => true); + if (error) return next(new HttpError(422, `Network error getting EC2 metadata: ${error.message}`)); + if (response.statusCode !== 200) return next(new HttpError(422, `Unable to get EC2 meta data. statusCode: ${response.status}`)); - if (result.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match')); + if (response.body !== req.body.providerToken) return next(new HttpError(422, 'Instance ID does not match')); - next(); - }); + next(); } else { next(); } diff --git a/src/routes/services.js b/src/routes/services.js index 6dbd927d0..854d0ef2f 100644 --- a/src/routes/services.js +++ b/src/routes/services.js @@ -24,19 +24,18 @@ async function getAll(req, res, next) { next(new HttpSuccess(200, { services: result })); } -function get(req, res, next) { +async function get(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); req.clearTimeout(); - services.getServiceStatus(req.params.service, function (error, result) { - if (error) return next(BoxError.toHttpError(error)); + const [error, result] = await safe(services.getServiceStatus(req.params.service)); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(200, { service: result })); - }); + next(new HttpSuccess(200, { service: result })); } -function configure(req, res, next) { +async function configure(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); if (typeof req.body.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit must be a number')); @@ -50,92 +49,87 @@ function configure(req, res, next) { data.requireAdmin = req.body.requireAdmin; } - services.configureService(req.params.service, data, function (error) { - if (error) return next(BoxError.toHttpError(error)); + const [error] = await safe(services.configureService(req.params.service, data)); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(202, {})); - }); + next(new HttpSuccess(202, {})); } -function getLogs(req, res, next) { +async function getLogs(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); - var lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10; // we ignore last-event-id + const lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10; // we ignore last-event-id if (isNaN(lines)) return next(new HttpError(400, 'lines must be a number')); - var options = { + const options = { lines: lines, follow: false, format: req.query.format || 'json' }; - services.getServiceLogs(req.params.service, options, function (error, logStream) { - if (error) return next(BoxError.toHttpError(error)); + const [error, logStream] = await safe(services.getServiceLogs(req.params.service, options)); + if (error) return next(BoxError.toHttpError(error)); - res.writeHead(200, { - 'Content-Type': 'application/x-logs', - 'Content-Disposition': `attachment; filename="${req.params.service}.log"`, - 'Cache-Control': 'no-cache', - 'X-Accel-Buffering': 'no' // disable nginx buffering - }); - logStream.pipe(res); + res.writeHead(200, { + 'Content-Type': 'application/x-logs', + 'Content-Disposition': `attachment; filename="${req.params.service}.log"`, + 'Cache-Control': 'no-cache', + 'X-Accel-Buffering': 'no' // disable nginx buffering }); + logStream.pipe(res); } // this route is for streaming logs -function getLogStream(req, res, next) { +async function getLogStream(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); - var lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10; // we ignore last-event-id + const lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10; // we ignore last-event-id if (isNaN(lines)) return next(new HttpError(400, 'lines must be a valid number')); function sse(id, data) { return 'id: ' + id + '\ndata: ' + data + '\n\n'; } if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream')); - var options = { + const options = { lines: lines, follow: true, format: 'json' }; - services.getServiceLogs(req.params.service, options, function (error, logStream) { - if (error) return next(BoxError.toHttpError(error)); + const [error, logStream] = await safe(services.getServiceLogs(req.params.service, options)); + if (error) return next(BoxError.toHttpError(error)); - res.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive', - 'X-Accel-Buffering': 'no', // disable nginx buffering - 'Access-Control-Allow-Origin': '*' - }); - res.write('retry: 3000\n'); - res.on('close', logStream.close); - logStream.on('data', function (data) { - var obj = JSON.parse(data); - res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id - }); - logStream.on('end', res.end.bind(res)); - logStream.on('error', res.end.bind(res, null)); + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'X-Accel-Buffering': 'no', // disable nginx buffering + 'Access-Control-Allow-Origin': '*' }); + res.write('retry: 3000\n'); + res.on('close', logStream.close); + logStream.on('data', function (data) { + var obj = JSON.parse(data); + res.write(sse(obj.monotonicTimestamp, JSON.stringify(obj))); // send timestamp as id + }); + logStream.on('end', res.end.bind(res)); + logStream.on('error', res.end.bind(res, null)); } -function restart(req, res, next) { +async function restart(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); - services.restartService(req.params.service, function (error) { - if (error) return next(BoxError.toHttpError(error)); + const [error] = await safe(services.restartService(req.params.service)); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(202, {})); - }); + next(new HttpSuccess(202, {})); } -function rebuild(req, res, next) { +async function rebuild(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); - services.rebuildService(req.params.service, function (error) { - if (error) return next(BoxError.toHttpError(error)); + const [error] = await safe(services.rebuildService(req.params.service)); + if (error) return next(BoxError.toHttpError(error)); - next(new HttpSuccess(202, {})); - }); + next(new HttpSuccess(202, {})); } diff --git a/src/routes/test/users-test.js b/src/routes/test/users-test.js index 771e47e59..584fd4df1 100644 --- a/src/routes/test/users-test.js +++ b/src/routes/test/users-test.js @@ -5,7 +5,6 @@ 'use strict'; -const { execContainer } = require('../../docker.js'); const common = require('./common.js'), expect = require('expect.js'), superagent = require('superagent'), diff --git a/src/scheduler.js b/src/scheduler.js index b91824a8e..87ae8294b 100644 --- a/src/scheduler.js +++ b/src/scheduler.js @@ -8,13 +8,12 @@ exports = module.exports = { const apps = require('./apps.js'), assert = require('assert'), - async = require('async'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), CronJob = require('cron').CronJob, debug = require('debug')('box:scheduler'), docker = require('./docker.js'), - util = require('util'), + safe = require('safetydance'), _ = require('underscore'); // appId -> { containerId, schedulerConfig (manifest), cronjobs } @@ -32,42 +31,37 @@ function resumeJobs(appId) { gSuspendedAppIds.delete(appId); } -function runTask(appId, taskName, callback) { +async function runTask(appId, taskName) { assert.strictEqual(typeof appId, 'string'); assert.strictEqual(typeof taskName, 'string'); - assert.strictEqual(typeof callback, 'function'); const JOB_MAX_TIME = 30 * 60 * 1000; // 30 minutes const containerName = `${appId}-${taskName}`; - if (gSuspendedAppIds.has(appId)) return callback(); + if (gSuspendedAppIds.has(appId)) return; - util.callbackify(apps.get)(appId, function (error, app) { - if (error) return callback(error); - if (!app) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); + const app = await apps.get(appId); + if (!app) throw new BoxError(BoxError.NOT_FOUND, 'App not found'); - if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING || app.health !== apps.HEALTH_HEALTHY) return callback(); + if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING || app.health !== apps.HEALTH_HEALTHY) return; - docker.inspectByName(containerName, function (error, data) { - if (!error && data && data.State.Running === true) { - const jobStartTime = new Date(data.State.StartedAt); // iso 8601 - if (new Date() - jobStartTime < JOB_MAX_TIME) return callback(); - } + const [error, data] = await safe(docker.inspectByName(containerName)); + if (!error && data && data.State.Running === true) { + const jobStartTime = new Date(data.State.StartedAt); // iso 8601 + if (new Date() - jobStartTime < JOB_MAX_TIME) return; + } - docker.restartContainer(containerName, callback); - }); - }); + await docker.restartContainer(containerName); } -function createJobs(app, schedulerConfig, callback) { +async function createJobs(app, schedulerConfig) { assert.strictEqual(typeof app, 'object'); assert(schedulerConfig && typeof schedulerConfig === 'object'); - assert.strictEqual(typeof callback, 'function'); const appId = app.id; - let jobs = { }; + const jobs = { }; - async.eachSeries(Object.keys(schedulerConfig), function (taskName, iteratorDone) { + for (const taskName of Object.keys(schedulerConfig)) { const task = schedulerConfig[taskName]; const randomSecond = Math.floor(60*Math.random()); // don't start all crons to decrease memory pressure const cronTime = (constants.TEST ? '*/5 ' : `${randomSecond} `) + task.schedule; // time ticks faster in tests @@ -77,97 +71,79 @@ function createJobs(app, schedulerConfig, callback) { // stopJobs only deletes jobs since previous run. This means that when box code restarts, none of the containers // are removed. The deleteContainer here ensures we re-create the cron containers with the latest config - docker.deleteContainer(containerName, function ( /* ignoredError */) { - docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error) { - if (error && error.reason !== BoxError.ALREADY_EXISTS) return iteratorDone(error); + await safe(docker.deleteContainer(containerName)); // ignore error + const [error] = await safe(docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */)); + if (error && error.reason !== BoxError.ALREADY_EXISTS) continue; - debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`); + debug(`createJobs: ${taskName} (${app.fqdn}) will run in container ${containerName}`); - var cronJob = new CronJob({ - cronTime: cronTime, // at this point, the pattern has been validated - onTick: () => runTask(appId, taskName, (error) => { // put the app id in closure, so we don't use the outdated app object by mistake - if (error) debug(`could not run task ${taskName} : ${error.message}`); - }), - start: true - }); - - jobs[taskName] = cronJob; - - iteratorDone(); - }); + const cronJob = new CronJob({ + cronTime: cronTime, // at this point, the pattern has been validated + onTick: async () => { + const [error] = await safe(runTask(appId, taskName)); // put the app id in closure, so we don't use the outdated app object by mistake + if (error) debug(`could not run task ${taskName} : ${error.message}`); + }, + start: true }); - }, function (error) { - callback(error, jobs); - }); + + jobs[taskName] = cronJob; + } + + return jobs; } -function stopJobs(appId, appState, callback) { +async function stopJobs(appId, appState) { assert.strictEqual(typeof appId, 'string'); assert.strictEqual(typeof appState, 'object'); - assert.strictEqual(typeof callback, 'function'); - if (!appState) return callback(); + if (!appState) return; - async.eachSeries(Object.keys(appState.schedulerConfig), function (taskName, iteratorDone) { + for (const taskName of Object.keys(appState.schedulerConfig)) { if (appState.cronJobs && appState.cronJobs[taskName]) appState.cronJobs[taskName].stop(); const containerName = `${appId}-${taskName}`; - docker.deleteContainer(containerName, function (error) { - if (error) debug(`stopJobs: failed to delete task container with name ${containerName} : ${error.message}`); - - iteratorDone(); - }); - }, callback); + const [error] = await safe(docker.deleteContainer(containerName)); + if (error) debug(`stopJobs: failed to delete task container with name ${containerName} : ${error.message}`); + } } -function sync() { +async function sync() { if (constants.TEST) return; - util.callbackify(apps.list)(function (error, allApps) { - if (error) return debug(`sync: error getting app list. ${error.message}`); + const allApps = await apps.list(); - var allAppIds = allApps.map(function (app) { return app.id; }); - var removedAppIds = _.difference(Object.keys(gState), allAppIds); - if (removedAppIds.length !== 0) debug(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`); + const allAppIds = allApps.map(function (app) { return app.id; }); + const removedAppIds = _.difference(Object.keys(gState), allAppIds); + if (removedAppIds.length !== 0) debug(`sync: stopping jobs of removed apps ${JSON.stringify(removedAppIds)}`); - async.eachSeries(removedAppIds, function (appId, iteratorDone) { - debug(`sync: removing jobs of ${appId}`); - stopJobs(appId, gState[appId], iteratorDone); - }, function (error) { - if (error) debug(`sync: error stopping jobs of removed apps: ${error.message}`); + for (const appId of removedAppIds) { + debug(`sync: removing jobs of ${appId}`); + const [error] = await safe(stopJobs(appId, gState[appId])); + if (error) debug(`sync: error stopping jobs of removed app ${appId}: ${error.message}`); + } - gState = _.omit(gState, removedAppIds); + gState = _.omit(gState, removedAppIds); - async.eachSeries(allApps, function (app, iteratorDone) { - var appState = gState[app.id] || null; - var schedulerConfig = app.manifest.addons ? app.manifest.addons.scheduler : null; + for (const app of allApps) { + const appState = gState[app.id] || null; + const schedulerConfig = app.manifest.addons ? app.manifest.addons.scheduler : null; - if (!appState && !schedulerConfig) return iteratorDone(); // nothing to do - if (appState && appState.cronJobs) { // we had created jobs for this app previously - if (_.isEqual(appState.schedulerConfig, schedulerConfig) && appState.containerId === app.containerId) return iteratorDone(); // nothing changed - } + if (!appState && !schedulerConfig) continue; // nothing to do + if (appState && appState.cronJobs) { // we had created jobs for this app previously + if (_.isEqual(appState.schedulerConfig, schedulerConfig) && appState.containerId === app.containerId) continue; // nothing changed + } - debug(`sync: adding jobs of ${app.id} (${app.fqdn})`); + debug(`sync: adding jobs of ${app.id} (${app.fqdn})`); - stopJobs(app.id, appState, function (error) { - if (error) debug(`sync: error stopping jobs of ${app.id} : ${error.message}`); + const [error] = await safe(stopJobs(app.id, appState)); + if (error) debug(`sync: error stopping jobs of ${app.id} : ${error.message}`); - if (!schedulerConfig) { // updated app version removed scheduler addon - delete gState[app.id]; - return iteratorDone(); - } + if (!schedulerConfig) { // updated app version removed scheduler addon + delete gState[app.id]; + continue; + } - createJobs(app, schedulerConfig, function (error, cronJobs) { - if (error) return iteratorDone(error); // if docker is down, the next sync() will recreate everything for this app - - gState[app.id] = { containerId: app.containerId, schedulerConfig, cronJobs }; - - iteratorDone(); - }); - }); - }, function (error) { - if (error) return debug('sync: error creating jobs', error.message); - }); - }); - }); + const cronJobs = await createJobs(app, schedulerConfig); // if docker is down, the next sync() will recreate everything for this app + gState[app.id] = { containerId: app.containerId, schedulerConfig, cronJobs }; + } } diff --git a/src/services.js b/src/services.js index e07656086..928badb8a 100644 --- a/src/services.js +++ b/src/services.js @@ -3,7 +3,6 @@ exports = module.exports = { getServiceIds, getServiceStatus, - getServiceConfig, getServiceLogs, configureService, @@ -49,7 +48,7 @@ const addonConfigs = require('./addonconfigs.js'), os = require('os'), path = require('path'), paths = require('./paths.js'), - rimraf = require('rimraf'), + promiseRetry = require('./promise-retry.js'), safe = require('safetydance'), semver = require('semver'), settings = require('./settings.js'), @@ -57,21 +56,15 @@ const addonConfigs = require('./addonconfigs.js'), shell = require('./shell.js'), spawn = require('child_process').spawn, split = require('split'), + superagent = require('superagent'), request = require('request'), system = require('./system.js'), util = require('util'); -const settingsGetServicesConfig = util.callbackify(settings.getServicesConfig); - -const NOOP = function (app, options, callback) { return callback(); }; -const NOOP_CALLBACK = function (error) { if (error) debug(error); }; +const NOOP = async function (/*app, options*/) {}; const RMADDONDIR_CMD = path.join(__dirname, 'scripts/rmaddondir.sh'); const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh'); -const setAddonConfig = util.callbackify(addonConfigs.set), - unsetAddonConfig = util.callbackify(addonConfigs.unset), - getAddonConfigByName = util.callbackify(addonConfigs.getByName); - // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost // teardown is destructive. app data stored with the addon is lost const ADDONS = { @@ -276,56 +269,50 @@ function dumpPath(addon, appId) { } } -function getContainerDetails(containerName, tokenEnvName, callback) { +async function getContainerDetails(containerName, tokenEnvName) { assert.strictEqual(typeof containerName, 'string'); assert.strictEqual(typeof tokenEnvName, 'string'); - assert.strictEqual(typeof callback, 'function'); - docker.inspect(containerName, function (error, result) { - if (error) return callback(error); + const result = await docker.inspect(containerName); - const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null); - if (!ip) return callback(new BoxError(BoxError.INACTIVE, `Error getting IP of ${containerName} service`)); + const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null); + if (!ip) throw new BoxError(BoxError.INACTIVE, `Error getting IP of ${containerName} service`); - // extract the cloudron token for auth - const env = safe.query(result, 'Config.Env', null); - if (!env) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error inspecting environment of ${containerName} service`)); - const tmp = env.find(function (e) { return e.indexOf(tokenEnvName) === 0; }); - if (!tmp) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`)); - const token = tmp.slice(tokenEnvName.length + 1); // +1 for the = sign - if (!token) return callback(new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`)); + // extract the cloudron token for auth + const env = safe.query(result, 'Config.Env', null); + if (!env) throw new BoxError(BoxError.DOCKER_ERROR, `Error inspecting environment of ${containerName} service`); + const tmp = env.find(function (e) { return e.indexOf(tokenEnvName) === 0; }); + if (!tmp) throw new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`); + const token = tmp.slice(tokenEnvName.length + 1); // +1 for the = sign + if (!token) throw new BoxError(BoxError.DOCKER_ERROR, `Error getting token of ${containerName} service`); - callback(null, { ip: ip, token: token, state: result.State }); - }); + return { ip: ip, token: token, state: result.State }; } -function containerStatus(containerName, tokenEnvName, callback) { +async function containerStatus(containerName, tokenEnvName) { assert.strictEqual(typeof containerName, 'string'); assert.strictEqual(typeof tokenEnvName, 'string'); - assert.strictEqual(typeof callback, 'function'); - getContainerDetails(containerName, tokenEnvName, function (error, addonDetails) { - if (error && (error.reason === BoxError.NOT_FOUND || error.reason === BoxError.INACTIVE)) return callback(null, { status: exports.SERVICE_STATUS_STOPPED }); - if (error) return callback(error); + const [error, addonDetails] = await safe(getContainerDetails(containerName, tokenEnvName)); + if (error && (error.reason === BoxError.NOT_FOUND || error.reason === BoxError.INACTIVE)) return { status: exports.SERVICE_STATUS_STOPPED }; + if (error) throw error; - request.get(`https://${addonDetails.ip}:3000/healthcheck?access_token=${addonDetails.token}`, { json: true, rejectUnauthorized: false, timeout: 20000 }, function (error, response) { - if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for ${containerName}: ${error.message}` }); - if (response.statusCode !== 200 || !response.body.status) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}` }); + const [networkError, response] = await safe(superagent.get(`https://${addonDetails.ip}:3000/healthcheck?access_token=${addonDetails.token}`) + .disableTLSCerts() + .timeout(20000) + .ok(() => true)); - docker.memoryUsage(containerName, function (error, result) { - if (error) return callback(error); + if (networkError) return { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for ${containerName}: ${networkError.message}` }; + if (response.status !== 200 || !response.body.status) return { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}` }; - var tmp = { - status: addonDetails.state.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, - memoryUsed: result.memory_stats.usage, - memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit), - healthcheck: response.body - }; + const result = await docker.memoryUsage(containerName); - callback(null, tmp); - }); - }); - }); + return { + status: addonDetails.state.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, + memoryUsed: result.memory_stats.usage, + memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit), + healthcheck: response.body + }; } async function getServiceIds() { @@ -339,48 +326,39 @@ async function getServiceIds() { return serviceIds; } -function getServiceConfig(id, callback) { +async function getServiceConfig(id) { assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof callback, 'function'); const [name, instance] = id.split(':'); if (!instance) { - settingsGetServicesConfig(function (error, servicesConfig) { - if (error) return callback(error); - - callback(null, servicesConfig[name] || {}); - }); - - return; + const servicesConfig = await settings.getServicesConfig(); + return servicesConfig[name] || {}; } - util.callbackify(apps.get)(instance, function (error, app) { - if (error) return callback(error); - if (!app) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); + const app = await apps.get(instance); + if (!app) throw new BoxError(BoxError.NOT_FOUND, 'App not found'); - callback(null, app.servicesConfig[name] || {}); - }); + return app.servicesConfig[name] || {}; } -function getServiceStatus(id, callback) { +async function getServiceStatus(id) { assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof callback, 'function'); const [name, instance ] = id.split(':'); let containerStatusFunc, service; if (instance) { service = APP_SERVICES[name]; - if (!service) return callback(new BoxError(BoxError.NOT_FOUND)); + if (!service) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); containerStatusFunc = service.status.bind(null, instance); } else if (SERVICES[name]) { service = SERVICES[name]; containerStatusFunc = service.status; } else { - return callback(new BoxError(BoxError.NOT_FOUND)); + throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); } - var tmp = { + const tmp = { name: name, status: null, memoryUsed: 0, @@ -390,72 +368,54 @@ function getServiceStatus(id, callback) { config: {} }; - containerStatusFunc(function (error, result) { - if (error) return callback(error); + const result = await containerStatusFunc(); + tmp.status = result.status; + tmp.memoryUsed = result.memoryUsed; + tmp.memoryPercent = result.memoryPercent; + tmp.error = result.error || null; + tmp.healthcheck = result.healthcheck || null; - tmp.status = result.status; - tmp.memoryUsed = result.memoryUsed; - tmp.memoryPercent = result.memoryPercent; - tmp.error = result.error || null; - tmp.healthcheck = result.healthcheck || null; + tmp.config = await getServiceConfig(id); - getServiceConfig(id, function (error, serviceConfig) { - if (error) return callback(error); + if (!tmp.config.memoryLimit && service.defaultMemoryLimit) { + tmp.config.memoryLimit = service.defaultMemoryLimit; + } - tmp.config = serviceConfig; - - if (!tmp.config.memoryLimit && service.defaultMemoryLimit) { - tmp.config.memoryLimit = service.defaultMemoryLimit; - } - - callback(null, tmp); - }); - }); + return tmp; } -function configureService(id, data, callback) { +async function configureService(id, data) { assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof data, 'object'); - assert.strictEqual(typeof callback, 'function'); const [name, instance ] = id.split(':'); if (instance) { - if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND)); + if (!APP_SERVICES[name]) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); - util.callbackify(apps.get)(instance, async function (error, app) { - if (error) return callback(error); - if (!app) return callback(new BoxError(BoxError.NOT_FOUND, 'App not found')); + const app = await apps.get(instance); + if (!app) throw new BoxError(BoxError.NOT_FOUND, 'App not found'); - const servicesConfig = app.servicesConfig; - servicesConfig[name] = data; + const servicesConfig = app.servicesConfig; + servicesConfig[name] = data; - const [updateError] = await safe(apps.update(instance, { servicesConfig })); - if (updateError) return callback(updateError); + await apps.update(instance, { servicesConfig }); - applyServiceConfig(id, data, callback); - }); + await applyServiceConfig(id, data); } else if (SERVICES[name]) { - settingsGetServicesConfig(function (error, servicesConfig) { - if (error) return callback(error); + const servicesConfig = await settings.getServicesConfig(); + servicesConfig[name] = data; - servicesConfig[name] = data; - - util.callbackify(settings.setServicesConfig)(servicesConfig, function (error) { - if (error) return callback(error); - - applyServiceConfig(id, data, callback); - }); - }); + await settings.setServicesConfig(servicesConfig); + await applyServiceConfig(id, data); } else { - return callback(new BoxError(BoxError.NOT_FOUND)); + throw new BoxError(BoxError.NOT_FOUND, 'No such service'); } } -function getServiceLogs(id, options, callback) { +async function getServiceLogs(id, options) { assert.strictEqual(typeof id, 'string'); assert(options && typeof options === 'object'); - assert.strictEqual(typeof callback, 'function'); assert.strictEqual(typeof options.lines, 'number'); assert.strictEqual(typeof options.format, 'string'); @@ -464,14 +424,14 @@ function getServiceLogs(id, options, callback) { const [name, instance ] = id.split(':'); if (instance) { - if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND)); + if (!APP_SERVICES[name]) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); } else if (!SERVICES[name]) { - return callback(new BoxError(BoxError.NOT_FOUND)); + throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); } debug(`Getting logs for ${name}`); - var lines = options.lines, + const lines = options.lines, format = options.format || 'json', follow = options.follow; @@ -503,15 +463,15 @@ function getServiceLogs(id, options, callback) { args.push(path.join(paths.LOG_DIR, containerName, 'app.log')); } - var cp = spawn(cmd, args); + const cp = spawn(cmd, args); - var transformStream = split(function mapper(line) { + const transformStream = split(function mapper(line) { if (format !== 'json') return line + '\n'; - var data = line.split(' '); // logs are - var timestamp = (new Date(data[0])).getTime(); + const data = line.split(' '); // logs are + let timestamp = (new Date(data[0])).getTime(); if (isNaN(timestamp)) timestamp = 0; - var message = line.slice(data[0].length+1); + const message = line.slice(data[0].length+1); // ignore faulty empty logs if (!timestamp && !message) return; @@ -527,194 +487,174 @@ function getServiceLogs(id, options, callback) { cp.stdout.pipe(transformStream); - callback(null, transformStream); + return transformStream; } -function rebuildService(id, callback) { +async function rebuildService(id) { assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof callback, 'function'); // this attempts to recreate the service docker container if they don't exist but platform infra version is unchanged // passing an infra version of 'none' will not attempt to purge existing data, not sure if this is good or bad - getServiceConfig(id, function (error, serviceConfig) { - if (error) return callback(error); + const serviceConfig = await getServiceConfig(id); - if (id === 'turn') return startTurn({ version: 'none' }, serviceConfig, callback); - if (id === 'mongodb') return startMongodb({ version: 'none' }, callback); - if (id === 'postgresql') return startPostgresql({ version: 'none' }, callback); - if (id === 'mysql') return startMysql({ version: 'none' }, callback); - if (id === 'sftp') return sftp.rebuild(serviceConfig, { /* options */ }, callback); - if (id === 'graphite') return startGraphite({ version: 'none' }, serviceConfig, callback); + if (id === 'turn') return await startTurn({ version: 'none' }, serviceConfig); + if (id === 'mongodb') return await startMongodb({ version: 'none' }); + if (id === 'postgresql') return await startPostgresql({ version: 'none' }); + if (id === 'mysql') return await startMysql({ version: 'none' }); + if (id === 'sftp') return await sftp.rebuild(serviceConfig, { /* options */ }); + if (id === 'graphite') return await startGraphite({ version: 'none' }, serviceConfig); - // nothing to rebuild for now. - // TODO: mongo/postgresql/mysql need to be scaled down. - // TODO: missing redis container is not created - callback(); - }); + // nothing to rebuild for now. + // TODO: mongo/postgresql/mysql need to be scaled down. + // TODO: missing redis container is not created } -function restartService(id, callback) { +async function restartService(id) { assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof callback, 'function'); const [name, instance ] = id.split(':'); if (instance) { - if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND)); + if (!APP_SERVICES[name]) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); - return APP_SERVICES[name].restart(instance, callback); + await APP_SERVICES[name].restart(instance); } else if (SERVICES[name]) { - return SERVICES[name].restart(callback); + await SERVICES[name].restart(); } else { - return callback(new BoxError(BoxError.NOT_FOUND)); + throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); } } // in the future, we can refcount and lazy start global services -function startAppServices(app, callback) { +async function startAppServices(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); const instance = app.id; - async.eachSeries(Object.keys(app.manifest.addons || {}), function (addon, iteratorDone) { - if (!(addon in APP_SERVICES)) return iteratorDone(); + for (const addon of Object.keys(app.manifest.addons || {})) { + if (!(addon in APP_SERVICES)) continue; - APP_SERVICES[addon].start(instance, function (error) { // assume addons name is service name - // error ignored because we don't want "start app" to error. use can fix it from Services - if (error) debug(`startAppServices: ${addon}:${instance}`, error); - - iteratorDone(); - }); - }, callback); + const [error] = await safe(APP_SERVICES[addon].start(instance)); // assume addons name is service name + // error ignored because we don't want "start app" to error. use can fix it from Services + if (error) debug(`startAppServices: ${addon}:${instance}`, error); + } } // in the future, we can refcount and stop global services as well -function stopAppServices(app, callback) { +async function stopAppServices(app) { assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof callback, 'function'); const instance = app.id; - async.eachSeries(Object.keys(app.manifest.addons || {}), function (addon, iteratorDone) { - if (!(addon in APP_SERVICES)) return iteratorDone(); + for (const addon of Object.keys(app.manifest.addons || {})) { + if (!(addon in APP_SERVICES)) continue; - APP_SERVICES[addon].stop(instance, function (error) { // assume addons name is service name - // error ignored because we don't want "start app" to error. use can fix it from Services - if (error) debug(`stopAppServices: ${addon}:${instance}`, error); - - iteratorDone(); - }); - }, callback); + const [error] = await safe(APP_SERVICES[addon].stop(instance)); // assume addons name is service name + // error ignored because we don't want "start app" to error. use can fix it from Services + if (error) debug(`stopAppServices: ${addon}:${instance}`, error); + } } -function waitForContainer(containerName, tokenEnvName, callback) { +async function waitForContainer(containerName, tokenEnvName) { assert.strictEqual(typeof containerName, 'string'); assert.strictEqual(typeof tokenEnvName, 'string'); - assert.strictEqual(typeof callback, 'function'); debug(`Waiting for ${containerName}`); - getContainerDetails(containerName, tokenEnvName, function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails(containerName, tokenEnvName); - async.retry({ times: 10, interval: 15000 }, function (retryCallback) { - request.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`, { json: true, rejectUnauthorized: false, timeout: 5000 }, function (error, response) { - if (error) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${error.message}`)); - if (response.statusCode !== 200 || !response.body.status) return retryCallback(new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.statusCode} message: ${response.body.message}`)); + await promiseRetry({ times: 10, interval: 15000 }, async () => { + const [networkError, response] = await safe(superagent.get(`https://${result.ip}:3000/healthcheck?access_token=${result.token}`) + .timeout(5000) + .disableTLSCerts() + .ok(() => true)); - retryCallback(null); - }); - }, callback); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error waiting for ${containerName}: ${networkError.message}`); + if (response.status !== 200 || !response.body.status) throw new BoxError(BoxError.ADDONS_ERROR, `Error waiting for ${containerName}. Status code: ${response.status} message: ${response.body.message}`); }); } -function setupAddons(app, addons, callback) { +async function setupAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - assert.strictEqual(typeof callback, 'function'); - if (!addons) return callback(null); + if (!addons) return; debugApp(app, 'setupAddons: Setting up %j', Object.keys(addons)); - async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) { - if (!(addon in ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`)); + for (const addon of Object.keys(addons)) { + if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); debugApp(app, 'Setting up addon %s with options %j', addon, addons[addon]); - ADDONS[addon].setup(app, addons[addon], iteratorCallback); - }, callback); + await ADDONS[addon].setup(app, addons[addon]); + } } -function teardownAddons(app, addons, callback) { +async function teardownAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - assert.strictEqual(typeof callback, 'function'); - if (!addons) return callback(null); + if (!addons) return; debugApp(app, 'teardownAddons: Tearing down %j', Object.keys(addons)); - async.eachSeries(Object.keys(addons), function iterator(addon, iteratorCallback) { - if (!(addon in ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`)); + for (const addon of Object.keys(addons)) { + if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); debugApp(app, 'Tearing down addon %s with options %j', addon, addons[addon]); - ADDONS[addon].teardown(app, addons[addon], iteratorCallback); - }, callback); + await ADDONS[addon].teardown(app, addons[addon]); + } } -function backupAddons(app, addons, callback) { +async function backupAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'backupAddons'); - if (!addons) return callback(null); + if (!addons) return; debugApp(app, 'backupAddons: Backing up %j', Object.keys(addons)); - async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) { - if (!(addon in ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`)); + for (const addon of Object.keys(addons)) { + if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); - ADDONS[addon].backup(app, addons[addon], iteratorCallback); - }, callback); + await ADDONS[addon].backup(app, addons[addon]); + } } -function clearAddons(app, addons, callback) { +async function clearAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'clearAddons'); - if (!addons) return callback(null); + if (!addons) return; debugApp(app, 'clearAddons: clearing %j', Object.keys(addons)); - async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) { - if (!(addon in ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`)); + for (const addon of Object.keys(addons)) { + if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); - ADDONS[addon].clear(app, addons[addon], iteratorCallback); - }, callback); + await ADDONS[addon].clear(app, addons[addon]); + } } -function restoreAddons(app, addons, callback) { +async function restoreAddons(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'restoreAddons'); - if (!addons) return callback(null); + if (!addons) return; debugApp(app, 'restoreAddons: restoring %j', Object.keys(addons)); - async.eachSeries(Object.keys(addons), function iterator (addon, iteratorCallback) { - if (!(addon in ADDONS)) return iteratorCallback(new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`)); + for (const addon of Object.keys(addons)) { + if (!(addon in ADDONS)) throw new BoxError(BoxError.NOT_FOUND, `No such addon: ${addon}`); - ADDONS[addon].restore(app, addons[addon], iteratorCallback); - }, callback); + await ADDONS[addon].restore(app, addons[addon]); + } } function importAppDatabase(app, addon, callback) { @@ -731,89 +671,73 @@ function importAppDatabase(app, addon, callback) { ], callback); } -function importDatabase(addon, callback) { +async function importDatabase(addon) { assert.strictEqual(typeof addon, 'string'); - assert.strictEqual(typeof callback, 'function'); debug(`importDatabase: Importing ${addon}`); - util.callbackify(apps.list)(function (error, allApps) { - if (error) return callback(error); + const allApps = await apps.list(); - async.eachSeries(allApps, function iterator (app, iteratorCallback) { - if (!(addon in app.manifest.addons)) return iteratorCallback(); // app doesn't use the addon + for (const app of allApps) { + if (!(addon in app.manifest.addons)) continue; // app doesn't use the addon - debug(`importDatabase: Importing addon ${addon} of app ${app.id}`); + debug(`importDatabase: Importing addon ${addon} of app ${app.id}`); - importAppDatabase(app, addon, async function (error) { - if (!error) return iteratorCallback(); + const [error] = await safe(importAppDatabase(app, addon)); + if (!error) continue; - debug(`importDatabase: Error importing ${addon} of app ${app.id}. Marking as errored`, error); - // FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db - // not clear, if repair workflow should be part of addon or per-app - const [updateError] = await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } })); - iteratorCallback(updateError); - }); - }, function (error) { - safe.fs.unlinkSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`)); // clean up for future migrations + debug(`importDatabase: Error importing ${addon} of app ${app.id}. Marking as errored`, error); + // FIXME: there is no way to 'repair' if we are here. we need to make a separate apptask that re-imports db + // not clear, if repair workflow should be part of addon or per-app + await safe(apps.update(app.id, { installationState: apps.ISTATE_ERROR, error: { message: error.message } })); + } - callback(error); - }); - }); + safe.fs.unlinkSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`)); // clean up for future migrations } -function exportDatabase(addon, callback) { +async function exportDatabase(addon) { assert.strictEqual(typeof addon, 'string'); - assert.strictEqual(typeof callback, 'function'); debug(`exportDatabase: Exporting ${addon}`); if (fs.existsSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`))) { debug(`exportDatabase: Already exported addon ${addon} in previous run`); - return callback(null); + return; } - util.callbackify(apps.list)(function (error, allApps) { - if (error) return callback(error); + const allApps = await apps.list(); - async.eachSeries(allApps, function iterator (app, iteratorCallback) { - if (!app.manifest.addons || !(addon in app.manifest.addons)) return iteratorCallback(); // app doesn't use the addon - if (app.installationState === apps.ISTATE_ERROR) return iteratorCallback(); // missing db causes crash in old app addon containers + for (const app of allApps) { + if (!app.manifest.addons || !(addon in app.manifest.addons)) continue; // app doesn't use the addon + if (app.installationState === apps.ISTATE_ERROR) continue; // missing db causes crash in old app addon containers - debug(`exportDatabase: Exporting addon ${addon} of app ${app.id}`); + debug(`exportDatabase: Exporting addon ${addon} of app ${app.id}`); - ADDONS[addon].backup(app, app.manifest.addons[addon], function (error) { - if (error) { - debug(`exportDatabase: Error exporting ${addon} of app ${app.id}.`, error); - // for errored apps, we can ignore if export had an error - return iteratorCallback(app.installationState === apps.ISTATE_ERROR ? null : error); - } + const [error] = await safe(ADDONS[addon].backup(app, app.manifest.addons[addon])); + if (error) { + debug(`exportDatabase: Error exporting ${addon} of app ${app.id}.`, error); + // for errored apps, we can ignore if export had an error + if (app.installationState === apps.ISTATE_ERROR) continue; + throw error; + } + } - iteratorCallback(); - }); - }, function (error) { - if (error) return callback(error); - - async.series([ - (done) => fs.writeFile(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`), '', 'utf8', done), - // note: after this point, we are restart safe. it's ok if the box code crashes at this point - (done) => shell.exec(`exportDatabase - remove${addon}`, `docker rm -f ${addon}`, done), // what if db writes something when quitting ... - (done) => shell.sudo(`exportDatabase - removeAddonDir${addon}`, [ RMADDONDIR_CMD, addon ], {}, done) // ready to start afresh - ], callback); - }); - }); + safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, `exported-${addon}`), '', 'utf8'); + if (safe.error) throw BoxError(BoxError.FS_ERROR, 'Error writing export checkpoint file'); + // note: after this point, we are restart safe. it's ok if the box code crashes at this point + await shell.promises.exec(`exportDatabase - remove${addon}`, `docker rm -f ${addon}`); // what if db writes something when quitting ... + await shell.promises.sudo(`exportDatabase - removeAddonDir${addon}`, [ RMADDONDIR_CMD, addon ], {}); // ready to start afresh } -function applyServiceConfig(id, serviceConfig, callback) { +async function applyServiceConfig(id, serviceConfig) { assert.strictEqual(typeof id, 'string'); assert.strictEqual(typeof serviceConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); const [name, instance] = id.split(':'); let containerName, memoryLimit; if (instance) { - if (!APP_SERVICES[name]) return callback(new BoxError(BoxError.NOT_FOUND)); + if (!APP_SERVICES[name]) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); containerName = `${name}-${instance}`; memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : APP_SERVICES[name].defaultMemoryLimit; @@ -821,67 +745,64 @@ function applyServiceConfig(id, serviceConfig, callback) { containerName = name; memoryLimit = serviceConfig && serviceConfig.memoryLimit ? serviceConfig.memoryLimit : SERVICES[name].defaultMemoryLimit; } else { - return callback(new BoxError(BoxError.NOT_FOUND)); + throw new BoxError(BoxError.NOT_FOUND, 'No such service'); } debug(`updateServiceConfig: ${containerName} ${JSON.stringify(serviceConfig)}`); const memory = system.getMemoryAllocation(memoryLimit); - docker.update(containerName, memory, memoryLimit, callback); + await docker.update(containerName, memory, memoryLimit); } -function startServices(existingInfra, callback) { +async function startServices(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - assert.strictEqual(typeof callback, 'function'); - settingsGetServicesConfig(function (error, servicesConfig) { - if (error) return callback(error); + const servicesConfig = await settings.getServicesConfig(); - let startFuncs = [ ]; + let startFuncs = [ ]; - // always start addons on any infra change, regardless of minor or major update - if (existingInfra.version !== infra.version) { - debug(`startServices: ${existingInfra.version} -> ${infra.version}. starting all services`); - startFuncs.push( - mail.startMail, // start this first to reduce email downtime - startTurn.bind(null, existingInfra, servicesConfig['turn'] || {}), - startMysql.bind(null, existingInfra), - startPostgresql.bind(null, existingInfra), - startMongodb.bind(null, existingInfra), - startRedis.bind(null, existingInfra), - startGraphite.bind(null, existingInfra, servicesConfig['graphite'] || {}), - sftp.start.bind(null, existingInfra, servicesConfig['sftp'] || {}), - ); - } else { - assert.strictEqual(typeof existingInfra.images, 'object'); + // always start addons on any infra change, regardless of minor or major update + if (existingInfra.version !== infra.version) { + debug(`startServices: ${existingInfra.version} -> ${infra.version}. starting all services`); + startFuncs.push( + mail.startMail, // start this first to reduce email downtime + startTurn.bind(null, existingInfra, servicesConfig['turn'] || {}), + startMysql.bind(null, existingInfra), + startPostgresql.bind(null, existingInfra), + startMongodb.bind(null, existingInfra), + startRedis.bind(null, existingInfra), + startGraphite.bind(null, existingInfra, servicesConfig['graphite'] || {}), + sftp.start.bind(null, existingInfra, servicesConfig['sftp'] || {}), + ); + } else { + assert.strictEqual(typeof existingInfra.images, 'object'); - if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(mail.startMail); // start this first to reduce email downtime - if (infra.images.turn.tag !== existingInfra.images.turn.tag) startFuncs.push(startTurn.bind(null, existingInfra, servicesConfig['turn'] || {})); - if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql.bind(null, existingInfra)); - if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql.bind(null, existingInfra)); - if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb.bind(null, existingInfra)); - if (infra.images.redis.tag !== existingInfra.images.redis.tag) startFuncs.push(startRedis.bind(null, existingInfra)); - if (infra.images.graphite.tag !== existingInfra.images.graphite.tag) startFuncs.push(startGraphite.bind(null, existingInfra, servicesConfig['graphite'] || {})); - if (infra.images.sftp.tag !== existingInfra.images.sftp.tag) startFuncs.push(sftp.start.bind(null, existingInfra, servicesConfig['sftp'] || {})); + if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(mail.startMail); // start this first to reduce email downtime + if (infra.images.turn.tag !== existingInfra.images.turn.tag) startFuncs.push(startTurn.bind(null, existingInfra, servicesConfig['turn'] || {})); + if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql.bind(null, existingInfra)); + if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql.bind(null, existingInfra)); + if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb.bind(null, existingInfra)); + if (infra.images.redis.tag !== existingInfra.images.redis.tag) startFuncs.push(startRedis.bind(null, existingInfra)); + if (infra.images.graphite.tag !== existingInfra.images.graphite.tag) startFuncs.push(startGraphite.bind(null, existingInfra, servicesConfig['graphite'] || {})); + if (infra.images.sftp.tag !== existingInfra.images.sftp.tag) startFuncs.push(sftp.start.bind(null, existingInfra, servicesConfig['sftp'] || {})); - debug('startServices: existing infra. incremental service create %j', startFuncs.map(function (f) { return f.name; })); - } + debug('startServices: existing infra. incremental service create %j', startFuncs.map(function (f) { return f.name; })); + } - async.series(startFuncs, function (error) { - if (error) return callback(error); + for (const func of startFuncs) { + await func(); + } - // we always start db containers with unlimited memory. we then scale them down per configuration - let updateFuncs = [ - applyServiceConfig.bind(null, 'mysql', servicesConfig['mysql'] || {}), - applyServiceConfig.bind(null, 'postgresql', servicesConfig['postgresql'] || {}), - applyServiceConfig.bind(null, 'mongodb', servicesConfig['mongodb'] || {}), - ]; + // we always start db containers with unlimited memory. we then scale them down per configuration + let updateFuncs = [ + applyServiceConfig.bind(null, 'mysql', servicesConfig['mysql'] || {}), + applyServiceConfig.bind(null, 'postgresql', servicesConfig['postgresql'] || {}), + applyServiceConfig.bind(null, 'mongodb', servicesConfig['mongodb'] || {}), + ]; - async.series(updateFuncs, NOOP_CALLBACK); // it's ok if applying service configs fails - - callback(); - }); - }); + for (const updateFunc of updateFuncs) { + safe(updateFunc()); // no waiting. and it's ok if applying service configs fails + } } async function getEnvironment(app) { @@ -898,11 +819,11 @@ function getContainerNamesSync(app, addons) { assert.strictEqual(typeof app, 'object'); assert(!addons || typeof addons === 'object'); - var names = [ ]; + let names = []; if (!addons) return names; - for (var addon in addons) { + for (const addon in addons) { switch (addon) { case 'scheduler': // names here depend on how scheduler.js creates containers @@ -915,135 +836,149 @@ function getContainerNamesSync(app, addons) { return names; } -function setupLocalStorage(app, options, callback) { +async function setupLocalStorage(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'setupLocalStorage'); const volumeDataDir = apps.getDataDir(app, app.dataDir); // reomve any existing volume in case it's bound with an old dataDir - async.series([ - docker.removeVolume.bind(null, `${app.id}-localstorage`), - docker.createVolume.bind(null, `${app.id}-localstorage`, volumeDataDir, { fqdn: app.fqdn, appId: app.id }) - ], callback); + await docker.removeVolume(`${app.id}-localstorage`); + await docker.createVolume(`${app.id}-localstorage`, volumeDataDir, { fqdn: app.fqdn, appId: app.id }); } -function clearLocalStorage(app, options, callback) { +async function clearLocalStorage(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'clearLocalStorage'); - docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: false }, callback); + await docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: false }); } -function teardownLocalStorage(app, options, callback) { +async function teardownLocalStorage(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'teardownLocalStorage'); - async.series([ - docker.clearVolume.bind(null, `${app.id}-localstorage`, { removeDirectory: true }), - docker.removeVolume.bind(null, `${app.id}-localstorage`) - ], callback); + await docker.clearVolume(`${app.id}-localstorage`, { removeDirectory: true }); + await docker.removeVolume(`${app.id}-localstorage`); } -function setupTurn(app, options, callback) { +async function setupTurn(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - const blobGet = util.callbackify(blobs.get); - blobGet(blobs.ADDON_TURN_SECRET, function (error, turnSecret) { - if (error) return callback(error); - if (!turnSecret) return callback(new BoxError(BoxError.ADDONS_ERROR, 'Turn secret is missing')); + const turnSecret = await blobs.get(blobs.ADDON_TURN_SECRET); + if (!turnSecret) throw new BoxError(BoxError.ADDONS_ERROR, 'Turn secret is missing'); - const env = [ - { name: 'CLOUDRON_STUN_SERVER', value: settings.dashboardFqdn() }, - { name: 'CLOUDRON_STUN_PORT', value: '3478' }, - { name: 'CLOUDRON_STUN_TLS_PORT', value: '5349' }, - { name: 'CLOUDRON_TURN_SERVER', value: settings.dashboardFqdn() }, - { name: 'CLOUDRON_TURN_PORT', value: '3478' }, - { name: 'CLOUDRON_TURN_TLS_PORT', value: '5349' }, - { name: 'CLOUDRON_TURN_SECRET', value: turnSecret } - ]; + const env = [ + { name: 'CLOUDRON_STUN_SERVER', value: settings.dashboardFqdn() }, + { name: 'CLOUDRON_STUN_PORT', value: '3478' }, + { name: 'CLOUDRON_STUN_TLS_PORT', value: '5349' }, + { name: 'CLOUDRON_TURN_SERVER', value: settings.dashboardFqdn() }, + { name: 'CLOUDRON_TURN_PORT', value: '3478' }, + { name: 'CLOUDRON_TURN_TLS_PORT', value: '5349' }, + { name: 'CLOUDRON_TURN_SECRET', value: turnSecret } + ]; - debugApp(app, 'Setting up TURN'); + debugApp(app, 'Setting up TURN'); - setAddonConfig(app.id, 'turn', env, callback); - }); + await addonConfigs.set(app.id, 'turn', env); } -function teardownTurn(app, options, callback) { +async function startTurn(existingInfra, serviceConfig) { + assert.strictEqual(typeof existingInfra, 'object'); + assert.strictEqual(typeof serviceConfig, 'object'); + + const tag = infra.images.turn.tag; + const memoryLimit = serviceConfig.memoryLimit || SERVICES['turn'].defaultMemoryLimit; + const memory = system.getMemoryAllocation(memoryLimit); + const realm = settings.dashboardFqdn(); + + const turnSecret = await blobs.get(blobs.ADDON_TURN_SECRET); + if (!turnSecret) throw new BoxError(BoxError.ADDONS_ERROR, 'Turn secret is missing'); + + // this exports 3478/tcp, 5349/tls and 50000-51000/udp. note that this runs on the host network! + const cmd = `docker run --restart=always -d --name="turn" \ + --hostname turn \ + --net host \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag=turn \ + -m ${memory} \ + --memory-swap ${memoryLimit} \ + --dns 172.18.0.1 \ + --dns-search=. \ + -e CLOUDRON_TURN_SECRET="${turnSecret}" \ + -e CLOUDRON_REALM="${realm}" \ + --label isCloudronManaged=true \ + --read-only -v /tmp -v /run "${tag}"`; + + await shell.promises.exec('stopTurn', 'docker stop turn || true'); + await shell.promises.exec('removeTurn', 'docker rm -f turn || true'); + await shell.promises.exec('startTurn', cmd); +} + +async function teardownTurn(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down TURN'); - unsetAddonConfig(app.id, 'turn', callback); + await addonConfigs.unset(app.id, 'turn'); } -function setupEmail(app, options, callback) { +async function setupEmail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - const listDomainsFunc = util.callbackify(mail.listDomains); - - listDomainsFunc(function (error, mailDomains) { - if (error) return callback(error); - - const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(','); - - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - - // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation) - var env = [ - { name: `${envPrefix}MAIL_SMTP_SERVER`, value: 'mail' }, - { name: `${envPrefix}MAIL_SMTP_PORT`, value: '2525' }, - { name: `${envPrefix}MAIL_IMAP_SERVER`, value: 'mail' }, - { name: `${envPrefix}MAIL_IMAP_PORT`, value: '9993' }, - { name: `${envPrefix}MAIL_SIEVE_SERVER`, value: 'mail' }, - { name: `${envPrefix}MAIL_SIEVE_PORT`, value: '4190' }, - { name: `${envPrefix}MAIL_DOMAIN`, value: app.domain }, - { name: `${envPrefix}MAIL_DOMAINS`, value: mailInDomains }, - { name: 'CLOUDRON_MAIL_SERVER_HOST', value: settings.mailFqdn() }, // this is also a hint to reconfigure on mail server name change - { name: `${envPrefix}LDAP_MAILBOXES_BASE_DN`, value: 'ou=mailboxes,dc=cloudron' } - ]; - - debugApp(app, 'Setting up Email'); - - setAddonConfig(app.id, 'email', env, callback); - }); -} - -function teardownEmail(app, options, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - - debugApp(app, 'Tearing down Email'); - - unsetAddonConfig(app.id, 'email', callback); -} - -function setupLdap(app, options, callback) { - assert.strictEqual(typeof app, 'object'); - assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - - if (!app.sso) return callback(null); + const mailDomains = await mail.listDomains(); + const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(','); const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - var env = [ + // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation) + const env = [ + { name: `${envPrefix}MAIL_SMTP_SERVER`, value: 'mail' }, + { name: `${envPrefix}MAIL_SMTP_PORT`, value: '2525' }, + { name: `${envPrefix}MAIL_IMAP_SERVER`, value: 'mail' }, + { name: `${envPrefix}MAIL_IMAP_PORT`, value: '9993' }, + { name: `${envPrefix}MAIL_SIEVE_SERVER`, value: 'mail' }, + { name: `${envPrefix}MAIL_SIEVE_PORT`, value: '4190' }, + { name: `${envPrefix}MAIL_DOMAIN`, value: app.domain }, + { name: `${envPrefix}MAIL_DOMAINS`, value: mailInDomains }, + { name: 'CLOUDRON_MAIL_SERVER_HOST', value: settings.mailFqdn() }, // this is also a hint to reconfigure on mail server name change + { name: `${envPrefix}LDAP_MAILBOXES_BASE_DN`, value: 'ou=mailboxes,dc=cloudron' } + ]; + + debugApp(app, 'Setting up Email'); + + await addonConfigs.set(app.id, 'email', env); +} + +async function teardownEmail(app, options) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof options, 'object'); + + debugApp(app, 'Tearing down Email'); + + await addonConfigs.unset(app.id, 'email'); +} + +async function setupLdap(app, options) { + assert.strictEqual(typeof app, 'object'); + assert.strictEqual(typeof options, 'object'); + + if (!app.sso) return; + + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + + const env = [ { name: `${envPrefix}LDAP_SERVER`, value: '172.18.0.1' }, { name: 'CLOUDRON_LDAP_HOST', value: '172.18.0.1' }, // to keep things in sync with the database _HOST vars { name: `${envPrefix}LDAP_PORT`, value: '' + constants.LDAP_PORT }, @@ -1056,97 +991,88 @@ function setupLdap(app, options, callback) { debugApp(app, 'Setting up LDAP'); - setAddonConfig(app.id, 'ldap', env, callback); + await addonConfigs.set(app.id, 'ldap', env); } -function teardownLdap(app, options, callback) { +async function teardownLdap(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down LDAP'); - unsetAddonConfig(app.id, 'ldap', callback); + await addonConfigs.unset(app.id, 'ldap'); } -function setupSendMail(app, options, callback) { +async function setupSendMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up SendMail'); const disabled = app.manifest.addons.sendmail.optional && !app.enableMailbox; - if (disabled) return setAddonConfig(app.id, 'sendmail', [], callback); + if (disabled) return await addonConfigs.set(app.id, 'sendmail', []); - getAddonConfigByName(app.id, 'sendmail', '%MAIL_SMTP_PASSWORD', function (error, existingPassword) { - if (error) return callback(error); + const existingPassword = await addonConfigs.getByName(app.id, 'sendmail', '%MAIL_SMTP_PASSWORD'); - const password = existingPassword || hat(4 * 48); // see box#565 for password length + const password = existingPassword || hat(4 * 48); // see box#565 for password length - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - const env = [ - { name: `${envPrefix}MAIL_SMTP_SERVER`, value: 'mail' }, - { name: `${envPrefix}MAIL_SMTP_PORT`, value: '2525' }, - { name: `${envPrefix}MAIL_SMTPS_PORT`, value: '2465' }, - { name: `${envPrefix}MAIL_STARTTLS_PORT`, value: '2587' }, - { name: `${envPrefix}MAIL_SMTP_USERNAME`, value: app.mailboxName + '@' + app.mailboxDomain }, - { name: `${envPrefix}MAIL_SMTP_PASSWORD`, value: password }, - { name: `${envPrefix}MAIL_FROM`, value: app.mailboxName + '@' + app.mailboxDomain }, - { name: `${envPrefix}MAIL_DOMAIN`, value: app.mailboxDomain } - ]; - debugApp(app, 'Setting sendmail addon config to %j', env); - setAddonConfig(app.id, 'sendmail', env, callback); - }); + const env = [ + { name: `${envPrefix}MAIL_SMTP_SERVER`, value: 'mail' }, + { name: `${envPrefix}MAIL_SMTP_PORT`, value: '2525' }, + { name: `${envPrefix}MAIL_SMTPS_PORT`, value: '2465' }, + { name: `${envPrefix}MAIL_STARTTLS_PORT`, value: '2587' }, + { name: `${envPrefix}MAIL_SMTP_USERNAME`, value: app.mailboxName + '@' + app.mailboxDomain }, + { name: `${envPrefix}MAIL_SMTP_PASSWORD`, value: password }, + { name: `${envPrefix}MAIL_FROM`, value: app.mailboxName + '@' + app.mailboxDomain }, + { name: `${envPrefix}MAIL_DOMAIN`, value: app.mailboxDomain } + ]; + debugApp(app, 'Setting sendmail addon config to %j', env); + await addonConfigs.set(app.id, 'sendmail', env); } -function teardownSendMail(app, options, callback) { +async function teardownSendMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down sendmail'); - unsetAddonConfig(app.id, 'sendmail', callback); + await addonConfigs.unset(app.id, 'sendmail'); } -function setupRecvMail(app, options, callback) { +async function setupRecvMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up recvmail'); - getAddonConfigByName(app.id, 'recvmail', '%MAIL_IMAP_PASSWORD', function (error, existingPassword) { - if (error) return callback(error); + const existingPassword = await addonConfigs.getByName(app.id, 'recvmail', '%MAIL_IMAP_PASSWORD'); - const password = existingPassword || hat(4 * 48); // see box#565 for password length + const password = existingPassword || hat(4 * 48); // see box#565 for password length - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - const env = [ - { name: `${envPrefix}MAIL_IMAP_SERVER`, value: 'mail' }, - { name: `${envPrefix}MAIL_IMAP_PORT`, value: '9993' }, - { name: `${envPrefix}MAIL_IMAP_USERNAME`, value: app.mailboxName + '@' + app.mailboxDomain }, - { name: `${envPrefix}MAIL_IMAP_PASSWORD`, value: password }, - { name: `${envPrefix}MAIL_TO`, value: app.mailboxName + '@' + app.mailboxDomain }, - { name: `${envPrefix}MAIL_DOMAIN`, value: app.mailboxDomain } - ]; + const env = [ + { name: `${envPrefix}MAIL_IMAP_SERVER`, value: 'mail' }, + { name: `${envPrefix}MAIL_IMAP_PORT`, value: '9993' }, + { name: `${envPrefix}MAIL_IMAP_USERNAME`, value: app.mailboxName + '@' + app.mailboxDomain }, + { name: `${envPrefix}MAIL_IMAP_PASSWORD`, value: password }, + { name: `${envPrefix}MAIL_TO`, value: app.mailboxName + '@' + app.mailboxDomain }, + { name: `${envPrefix}MAIL_DOMAIN`, value: app.mailboxDomain } + ]; - debugApp(app, 'Setting sendmail addon config to %j', env); - setAddonConfig(app.id, 'recvmail', env, callback); - }); + debugApp(app, 'Setting sendmail addon config to %j', env); + await addonConfigs.set(app.id, 'recvmail', env); } -function teardownRecvMail(app, options, callback) { +async function teardownRecvMail(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Tearing down recvmail'); - unsetAddonConfig(app.id, 'recvmail', callback); + await addonConfigs.unset(app.id, 'recvmail'); } function mysqlDatabaseName(appId) { @@ -1157,9 +1083,8 @@ function mysqlDatabaseName(appId) { return md5sum.digest('hex').substring(0, 16); // max length of mysql usernames is 16 } -function startMysql(existingInfra, callback) { +async function startMysql(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - assert.strictEqual(typeof callback, 'function'); const tag = infra.images.mysql.tag; const dataDir = paths.PLATFORM_DATA_DIR; @@ -1168,204 +1093,186 @@ function startMysql(existingInfra, callback) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mysql.tag, tag); - if (upgrading) debug('startMysql: mysql will be upgraded'); - const upgradeFunc = upgrading ? exportDatabase.bind(null, 'mysql') : (next) => next(); + if (upgrading) { + debug('startMysql: mysql will be upgraded'); + await exportDatabase('mysql'); + } - upgradeFunc(function (error) { - if (error) return callback(error); + // memory options are applied dynamically. import requires all the memory we can get + const cmd = `docker run --restart=always -d --name="mysql" \ + --hostname mysql \ + --net cloudron \ + --net-alias mysql \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag=mysql \ + --dns 172.18.0.1 \ + --dns-search=. \ + -e CLOUDRON_MYSQL_TOKEN=${cloudronToken} \ + -e CLOUDRON_MYSQL_ROOT_HOST=172.18.0.1 \ + -e CLOUDRON_MYSQL_ROOT_PASSWORD=${rootPassword} \ + -v "${dataDir}/mysql:/var/lib/mysql" \ + --label isCloudronManaged=true \ + --cap-add SYS_NICE \ + --read-only -v /tmp -v /run "${tag}"`; - // memory options are applied dynamically. import requires all the memory we can get - const cmd = `docker run --restart=always -d --name="mysql" \ - --hostname mysql \ - --net cloudron \ - --net-alias mysql \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag=mysql \ - --dns 172.18.0.1 \ - --dns-search=. \ - -e CLOUDRON_MYSQL_TOKEN=${cloudronToken} \ - -e CLOUDRON_MYSQL_ROOT_HOST=172.18.0.1 \ - -e CLOUDRON_MYSQL_ROOT_PASSWORD=${rootPassword} \ - -v "${dataDir}/mysql:/var/lib/mysql" \ - --label isCloudronManaged=true \ - --cap-add SYS_NICE \ - --read-only -v /tmp -v /run "${tag}"`; + await shell.promises.exec('stopMysql', 'docker stop mysql || true'); + await shell.promises.exec('removeMysql', 'docker rm -f mysql || true'); + await shell.promises.exec('startMysql', cmd); - async.series([ - shell.exec.bind(null, 'stopMysql', 'docker stop mysql || true'), - shell.exec.bind(null, 'removeMysql', 'docker rm -f mysql || true'), - shell.exec.bind(null, 'startMysql', cmd) - ], function (error) { - if (error) return callback(error); + await waitForContainer('mysql', 'CLOUDRON_MYSQL_TOKEN'); - waitForContainer('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error) { - if (error) return callback(error); - if (!upgrading) return callback(null); - - importDatabase('mysql', callback); - }); - }); - }); + if (upgrading) await importDatabase('mysql'); } -function setupMySql(app, options, callback) { +async function setupMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up mysql'); - getAddonConfigByName(app.id, 'mysql', '%MYSQL_PASSWORD', function (error, existingPassword) { - if (error) return callback(error); + const existingPassword = await addonConfigs.getByName(app.id, 'mysql', '%MYSQL_PASSWORD'); - const tmp = mysqlDatabaseName(app.id); + const tmp = mysqlDatabaseName(app.id); - const data = { - database: tmp, - prefix: tmp, - username: tmp, - password: existingPassword || hat(4 * 48) // see box#362 for password length - }; + const data = { + database: tmp, + prefix: tmp, + username: tmp, + password: existingPassword || hat(4 * 48) // see box#362 for password length + }; - getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); - request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${error.message}`)); - if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `?access_token=${result.token}`) + .disableTLSCerts() + .send(data) + .ok(() => true)); - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mysql: ${networkError.message}`); + if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up mysql. Status code: ${response.status} message: ${response.body.message}`); - var env = [ - { name: `${envPrefix}MYSQL_USERNAME`, value: data.username }, - { name: `${envPrefix}MYSQL_PASSWORD`, value: data.password }, - { name: `${envPrefix}MYSQL_HOST`, value: 'mysql' }, - { name: `${envPrefix}MYSQL_PORT`, value: '3306' } - ]; + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - if (options.multipleDatabases) { - env = env.concat({ name: `${envPrefix}MYSQL_DATABASE_PREFIX`, value: `${data.prefix}_` }); - } else { - env = env.concat( - { name: `${envPrefix}MYSQL_URL`, value: `mysql://${data.username}:${data.password}@mysql/${data.database}` }, - { name: `${envPrefix}MYSQL_DATABASE`, value: data.database } - ); - } + let env = [ + { name: `${envPrefix}MYSQL_USERNAME`, value: data.username }, + { name: `${envPrefix}MYSQL_PASSWORD`, value: data.password }, + { name: `${envPrefix}MYSQL_HOST`, value: 'mysql' }, + { name: `${envPrefix}MYSQL_PORT`, value: '3306' } + ]; - debugApp(app, 'Setting mysql addon config to %j', env); - setAddonConfig(app.id, 'mysql', env, callback); - }); - }); - }); + if (options.multipleDatabases) { + env = env.concat({ name: `${envPrefix}MYSQL_DATABASE_PREFIX`, value: `${data.prefix}_` }); + } else { + env = env.concat( + { name: `${envPrefix}MYSQL_URL`, value: `mysql://${data.username}:${data.password}@mysql/${data.database}` }, + { name: `${envPrefix}MYSQL_DATABASE`, value: data.database } + ); + } + + debugApp(app, 'Setting mysql addon config to %j', env); + await addonConfigs.set(app.id, 'mysql', env); } -function clearMySql(app, options, callback) { +async function clearMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const database = mysqlDatabaseName(app.id); - getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); - request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/clear?access_token=${result.token}`) + .disableTLSCerts() + .ok(() => true)); - callback(); - }); - }); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mysql: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing mysql. Status code: ${response.status} message: ${response.body.message}`); } -function teardownMySql(app, options, callback) { +async function teardownMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const database = mysqlDatabaseName(app.id); const username = database; - getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); - request.delete(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.del(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}?access_token=${result.token}&username=${username}`) + .disableTLSCerts() + .ok(() => true)); - unsetAddonConfig(app.id, 'mysql', callback); - }); - }); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mysql. Status code: ${response.status} message: ${response.body.message}`); + + await addonConfigs.unset(app.id, 'mysql'); } -function pipeRequestToFile(url, filename, callback) { +async function pipeRequestToFile(url, filename) { assert.strictEqual(typeof url, 'string'); assert.strictEqual(typeof filename, 'string'); - assert.strictEqual(typeof callback, 'function'); - const writeStream = fs.createWriteStream(filename); + return new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(filename); - const done = once(function (error) { // the writeStream and the request can both error - if (error) writeStream.close(); - callback(error); - }); + const done = once(function (error) { // the writeStream and the request can both error + if (error) writeStream.close(); + if (error) reject(error); else resolve(); + }); - writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`))); + writeStream.on('error', (error) => done(new BoxError(BoxError.FS_ERROR, `Error writing to ${filename}: ${error.message}`))); - writeStream.on('open', function () { - // note: do not attach to post callback handler because this will buffer the entire reponse! - // see https://github.com/request/request/issues/2270 - const req = request.post(url, { rejectUnauthorized: false }); - req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc - req.on('response', function (response) { - if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`)); + writeStream.on('open', function () { + // note: do not attach to post callback handler because this will buffer the entire reponse! + // see https://github.com/request/request/issues/2270 + const req = request.post(url, { rejectUnauthorized: false }); + req.on('error', (error) => done(new BoxError(BoxError.NETWORK_ERROR, `Request error writing to ${filename}: ${error.message}`))); // network error, dns error, request errored in middle etc + req.on('response', function (response) { + if (response.statusCode !== 200) return done(new BoxError(BoxError.ADDONS_ERROR, `Unexpected response code when piping ${url}: ${response.statusCode} message: ${response.statusMessage} filename: ${filename}`)); - response.pipe(writeStream).on('finish', done); // this is hit after data written to disk + response.pipe(writeStream).on('finish', done); // this is hit after data written to disk + }); }); }); } -function backupMySql(app, options, callback) { +async function backupMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const database = mysqlDatabaseName(app.id); debugApp(app, 'Backing up mysql'); - getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); - const url = `https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/backup?access_token=${result.token}`; - pipeRequestToFile(url, dumpPath('mysql', app.id), callback); - }); + const url = `https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/backup?access_token=${result.token}`; + await pipeRequestToFile(url, dumpPath('mysql', app.id)); } -function restoreMySql(app, options, callback) { +async function restoreMySql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const database = mysqlDatabaseName(app.id); debugApp(app, 'restoreMySql'); - callback = once(callback); // protect from multiple returns with streams + const result = await getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN'); - getContainerDetails('mysql', 'CLOUDRON_MYSQL_TOKEN', function (error, result) { - if (error) return callback(error); + return new Promise((resolve, reject) => { + reject = once(reject); // protect from multiple returns with streams - var input = fs.createReadStream(dumpPath('mysql', app.id)); - input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`))); + const input = fs.createReadStream(dumpPath('mysql', app.id)); + input.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mysql: ${error.message}`))); const restoreReq = request.post(`https://${result.ip}:3000/` + (options.multipleDatabases ? 'prefixes' : 'databases') + `/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`)); + if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql: ${error.message}`)); + if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mysql. Status code: ${response.statusCode} message: ${response.body.message}`)); - callback(null); + resolve(); }); input.pipe(restoreReq); @@ -1377,9 +1284,8 @@ function postgreSqlNames(appId) { return { database: `db${appId}`, username: `user${appId}` }; } -function startPostgresql(existingInfra, callback) { +async function startPostgresql(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - assert.strictEqual(typeof callback, 'function'); const tag = infra.images.postgresql.tag; const dataDir = paths.PLATFORM_DATA_DIR; @@ -1388,219 +1294,155 @@ function startPostgresql(existingInfra, callback) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.postgresql.tag, tag); - if (upgrading) debug('startPostgresql: postgresql will be upgraded'); - const upgradeFunc = upgrading ? exportDatabase.bind(null, 'postgresql') : (next) => next(); + if (upgrading) { + debug('startPostgresql: postgresql will be upgraded'); + await exportDatabase('postgresql'); + } - upgradeFunc(function (error) { - if (error) return callback(error); + // memory options are applied dynamically. import requires all the memory we can get + const cmd = `docker run --restart=always -d --name="postgresql" \ + --hostname postgresql \ + --net cloudron \ + --net-alias postgresql \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag=postgresql \ + --dns 172.18.0.1 \ + --dns-search=. \ + -e CLOUDRON_POSTGRESQL_ROOT_PASSWORD="${rootPassword}" \ + -e CLOUDRON_POSTGRESQL_TOKEN="${cloudronToken}" \ + -v "${dataDir}/postgresql:/var/lib/postgresql" \ + --label isCloudronManaged=true \ + --read-only -v /tmp -v /run "${tag}"`; - // memory options are applied dynamically. import requires all the memory we can get - const cmd = `docker run --restart=always -d --name="postgresql" \ - --hostname postgresql \ - --net cloudron \ - --net-alias postgresql \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag=postgresql \ - --dns 172.18.0.1 \ - --dns-search=. \ - -e CLOUDRON_POSTGRESQL_ROOT_PASSWORD="${rootPassword}" \ - -e CLOUDRON_POSTGRESQL_TOKEN="${cloudronToken}" \ - -v "${dataDir}/postgresql:/var/lib/postgresql" \ - --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + await shell.promises.exec('stopPostgresql', 'docker stop postgresql || true'); + await shell.promises.exec('removePostgresql', 'docker rm -f postgresql || true'); + await shell.promises.exec('startPostgresql', cmd); - async.series([ - shell.exec.bind(null, 'stopPostgresql', 'docker stop postgresql || true'), - shell.exec.bind(null, 'removePostgresql', 'docker rm -f postgresql || true'), - shell.exec.bind(null, 'startPostgresql', cmd) - ], function (error) { - if (error) return callback(error); - - waitForContainer('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error) { - if (error) return callback(error); - if (!upgrading) return callback(null); - - importDatabase('postgresql', callback); - }); - }); - }); + await waitForContainer('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); + if (upgrading) await importDatabase('postgresql'); } -function setupPostgreSql(app, options, callback) { +async function setupPostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up postgresql'); const { database, username } = postgreSqlNames(app.id); - getAddonConfigByName(app.id, 'postgresql', '%POSTGRESQL_PASSWORD', function (error, existingPassword) { - if (error) return callback(error); + const existingPassword = await addonConfigs.getByName(app.id, 'postgresql', '%POSTGRESQL_PASSWORD'); - const data = { - database: database, - username: username, - password: existingPassword || hat(4 * 128), - locale: options.locale || 'C' - }; + const data = { + database: database, + username: username, + password: existingPassword || hat(4 * 128), + locale: options.locale || 'C' + }; - getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${error.message}`)); - if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases?access_token=${result.token}`) + .disableTLSCerts() + .send(data) + .ok(() => true)); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up postgresql: ${networkError.message}`); + if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up postgresql. Status code: ${response.status} message: ${response.body.message}`); - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - var env = [ - { name: `${envPrefix}POSTGRESQL_URL`, value: `postgres://${data.username}:${data.password}@postgresql/${data.database}` }, - { name: `${envPrefix}POSTGRESQL_USERNAME`, value: data.username }, - { name: `${envPrefix}POSTGRESQL_PASSWORD`, value: data.password }, - { name: `${envPrefix}POSTGRESQL_HOST`, value: 'postgresql' }, - { name: `${envPrefix}POSTGRESQL_PORT`, value: '5432' }, - { name: `${envPrefix}POSTGRESQL_DATABASE`, value: data.database } - ]; + const env = [ + { name: `${envPrefix}POSTGRESQL_URL`, value: `postgres://${data.username}:${data.password}@postgresql/${data.database}` }, + { name: `${envPrefix}POSTGRESQL_USERNAME`, value: data.username }, + { name: `${envPrefix}POSTGRESQL_PASSWORD`, value: data.password }, + { name: `${envPrefix}POSTGRESQL_HOST`, value: 'postgresql' }, + { name: `${envPrefix}POSTGRESQL_PORT`, value: '5432' }, + { name: `${envPrefix}POSTGRESQL_DATABASE`, value: data.database } + ]; - debugApp(app, 'Setting postgresql addon config to %j', env); - setAddonConfig(app.id, 'postgresql', env, callback); - }); - }); - }); + debugApp(app, 'Setting postgresql addon config to %j', env); + addonConfigs.set(app.id, 'postgresql', env); } -function clearPostgreSql(app, options, callback) { +async function clearPostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const { database, username } = postgreSqlNames(app.id); const locale = options.locale || 'C'; debugApp(app, 'Clearing postgresql'); - getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}&locale=${locale}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.statusCode} message: ${response.body.message}`)); - - callback(null); - }); - }); + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}&username=${username}&locale=${locale}`) + .disableTLSCerts() + .ok(() => true)); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing postgresql: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing postgresql. Status code: ${response.status} message: ${response.body.message}`); } -function teardownPostgreSql(app, options, callback) { +async function teardownPostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const { database, username } = postgreSqlNames(app.id); - getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.del(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}&username=${username}`) + .disableTLSCerts() + .ok(() => true)); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error tearing down postgresql: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down postgresql. Status code: ${response.status} message: ${response.body.message}`); - unsetAddonConfig(app.id, 'postgresql', callback); - }); - }); + await addonConfigs.unset(app.id, 'postgresql'); } -function backupPostgreSql(app, options, callback) { +async function backupPostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Backing up postgresql'); const { database } = postgreSqlNames(app.id); - getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - const url = `https://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`; - pipeRequestToFile(url, dumpPath('postgresql', app.id), callback); - }); + const url = `https://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`; + await pipeRequestToFile(url, dumpPath('postgresql', app.id)); } -function restorePostgreSql(app, options, callback) { +async function restorePostgreSql(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Restore postgresql'); const { database, username } = postgreSqlNames(app.id); - callback = once(callback); // protect from multiple returns with streams + const result = await getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); - getContainerDetails('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN', function (error, result) { - if (error) return callback(error); + return new Promise((resolve, reject) => { + resolve = once(resolve); // protect from multiple returns with streams - var input = fs.createReadStream(dumpPath('postgresql', app.id)); - input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`))); + const input = fs.createReadStream(dumpPath('postgresql', app.id)); + input.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring postgresql: ${error.message}`))); const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}&username=${username}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`)); + if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql: ${error.message}`)); + if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring postgresql. Status code: ${response.statusCode} message: ${response.body.message}`)); - callback(null); + resolve(); }); input.pipe(restoreReq); }); } -function startTurn(existingInfra, serviceConfig, callback) { +async function startMongodb(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - assert.strictEqual(typeof serviceConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); - - const tag = infra.images.turn.tag; - const memoryLimit = serviceConfig.memoryLimit || SERVICES['turn'].defaultMemoryLimit; - const memory = system.getMemoryAllocation(memoryLimit); - const realm = settings.dashboardFqdn(); - - const blobGet = util.callbackify(blobs.get); - blobGet(blobs.ADDON_TURN_SECRET, function (error, turnSecret) { - if (error) return callback(error); - if (!turnSecret) return callback(new BoxError(BoxError.ADDONS_ERROR, 'Turn secret is missing')); - - // this exports 3478/tcp, 5349/tls and 50000-51000/udp. note that this runs on the host network! - const cmd = `docker run --restart=always -d --name="turn" \ - --hostname turn \ - --net host \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag=turn \ - -m ${memory} \ - --memory-swap ${memoryLimit} \ - --dns 172.18.0.1 \ - --dns-search=. \ - -e CLOUDRON_TURN_SECRET="${turnSecret}" \ - -e CLOUDRON_REALM="${realm}" \ - --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; - - async.series([ - shell.exec.bind(null, 'stopTurn', 'docker stop turn || true'), - shell.exec.bind(null, 'removeTurn', 'docker rm -f turn || true'), - shell.exec.bind(null, 'startTurn', cmd) - ], callback); - }); -} - -function startMongodb(existingInfra, callback) { - assert.strictEqual(typeof existingInfra, 'object'); - assert.strictEqual(typeof callback, 'function'); const tag = infra.images.mongodb.tag; const dataDir = paths.PLATFORM_DATA_DIR; @@ -1609,197 +1451,164 @@ function startMongodb(existingInfra, callback) { const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.mongodb.tag, tag); - if (upgrading) debug('startMongodb: mongodb will be upgraded'); - const upgradeFunc = upgrading ? exportDatabase.bind(null, 'mongodb') : (next) => next(); + if (upgrading) { + debug('startMongodb: mongodb will be upgraded'); + await exportDatabase('mongodb'); + } - upgradeFunc(function (error) { - if (error) return callback(error); + // memory options are applied dynamically. import requires all the memory we can get + const cmd = `docker run --restart=always -d --name="mongodb" \ + --hostname mongodb \ + --net cloudron \ + --net-alias mongodb \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag=mongodb \ + --dns 172.18.0.1 \ + --dns-search=. \ + -e CLOUDRON_MONGODB_ROOT_PASSWORD="${rootPassword}" \ + -e CLOUDRON_MONGODB_TOKEN="${cloudronToken}" \ + -v "${dataDir}/mongodb:/var/lib/mongodb" \ + --label isCloudronManaged=true \ + --read-only -v /tmp -v /run "${tag}"`; - // memory options are applied dynamically. import requires all the memory we can get - const cmd = `docker run --restart=always -d --name="mongodb" \ - --hostname mongodb \ - --net cloudron \ - --net-alias mongodb \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag=mongodb \ - --dns 172.18.0.1 \ - --dns-search=. \ - -e CLOUDRON_MONGODB_ROOT_PASSWORD="${rootPassword}" \ - -e CLOUDRON_MONGODB_TOKEN="${cloudronToken}" \ - -v "${dataDir}/mongodb:/var/lib/mongodb" \ - --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + await shell.promises.exec('stopMongodb', 'docker stop mongodb || true'); + await shell.promises.exec('removeMongodb', 'docker rm -f mongodb || true'); + await shell.promises.exec('startMongodb', cmd); - async.series([ - shell.exec.bind(null, 'stopMongodb', 'docker stop mongodb || true'), - shell.exec.bind(null, 'removeMongodb', 'docker rm -f mongodb || true'), - shell.exec.bind(null, 'startMongodb', cmd) - ], function (error) { - if (error) return callback(error); - - waitForContainer('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error) { - if (error) return callback(error); - if (!upgrading) return callback(null); - - importDatabase('mongodb', callback); - }); - }); - }); + await waitForContainer('mongodb', 'CLOUDRON_MONGODB_TOKEN'); + if (upgrading) await importDatabase('mongodb'); } -function setupMongoDb(app, options, callback) { +async function setupMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up mongodb'); - getAddonConfigByName(app.id, 'mongodb', '%MONGODB_PASSWORD', function (error, existingPassword) { - if (error) return callback(error); + const existingPassword = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_PASSWORD'); + let database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); + database = database || hat(8 * 8); // 16 bytes. keep this short, so as to not overflow the 127 byte index length in MongoDB < 4.4 - getAddonConfigByName(app.id, 'mongodb', '%MONGODB_DATABASE', function (error, database) { - if (error) return callback(error); + const data = { + database, + username: app.id, + password: existingPassword || hat(4 * 128), + oplog: !!options.oplog + }; - database = database || hat(8 * 8); // 16 bytes. keep this short, so as to not overflow the 127 byte index length in MongoDB < 4.4 + const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); - const data = { - database, - username: app.id, - password: existingPassword || hat(4 * 128), - oplog: !!options.oplog - }; + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases?access_token=${result.token}`) + .disableTLSCerts() + .send(data) + .ok(() => true)); - getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) { - if (error) return callback(error); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${networkError.message}`); + if (response.status !== 201) throw new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.status} message: ${response.body.message}`); - request.post(`https://${result.ip}:3000/databases?access_token=${result.token}`, { rejectUnauthorized: false, json: data }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error setting up mongodb: ${error.message}`)); - if (response.statusCode !== 201) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error setting up mongodb. Status code: ${response.statusCode} message: ${response.body.message}`)); + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + const env = [ + { name: `${envPrefix}MONGODB_URL`, value : `mongodb://${data.username}:${data.password}@mongodb:27017/${data.database}` }, + { name: `${envPrefix}MONGODB_USERNAME`, value : data.username }, + { name: `${envPrefix}MONGODB_PASSWORD`, value: data.password }, + { name: `${envPrefix}MONGODB_HOST`, value : 'mongodb' }, + { name: `${envPrefix}MONGODB_PORT`, value : '27017' }, + { name: `${envPrefix}MONGODB_DATABASE`, value : data.database } + ]; - var env = [ - { name: `${envPrefix}MONGODB_URL`, value : `mongodb://${data.username}:${data.password}@mongodb:27017/${data.database}` }, - { name: `${envPrefix}MONGODB_USERNAME`, value : data.username }, - { name: `${envPrefix}MONGODB_PASSWORD`, value: data.password }, - { name: `${envPrefix}MONGODB_HOST`, value : 'mongodb' }, - { name: `${envPrefix}MONGODB_PORT`, value : '27017' }, - { name: `${envPrefix}MONGODB_DATABASE`, value : data.database } - ]; + if (options.oplog) { + env.push({ name: `${envPrefix}MONGODB_OPLOG_URL`, value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` }); + } - if (options.oplog) { - env.push({ name: `${envPrefix}MONGODB_OPLOG_URL`, value : `mongodb://${data.username}:${data.password}@mongodb:27017/local?authSource=${data.database}` }); - } - - debugApp(app, 'Setting mongodb addon config to %j', env); - setAddonConfig(app.id, 'mongodb', env, callback); - }); - }); - }); - }); + debugApp(app, 'Setting mongodb addon config to %j', env); + await addonConfigs.set(app.id, 'mongodb', env); } -function clearMongodb(app, options, callback) { +async function clearMongodb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); - getAddonConfigByName(app.id, 'mongodb', '%MONGODB_DATABASE', function (error, database) { - if (error) return callback(error); - if (!database) return callback(new BoxError(BoxError.NOT_FOUND, 'Error clearing mongodb. No database')); + const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); + if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error clearing mongodb. No database'); - request.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/databases/${database}/clear?access_token=${result.token}`) + .disableTLSCerts() + .ok(() => true)); - callback(); - }); - }); - }); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing mongodb: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing mongodb. Status code: ${response.status} message: ${response.body.message}`); } -function teardownMongoDb(app, options, callback) { +async function teardownMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); - getAddonConfigByName(app.id, 'mongodb', '%MONGODB_DATABASE', function (error, database) { - if (error) return callback(error); - if (!database) return callback(null); + const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); + if (!database) return; - request.delete(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.del(`https://${result.ip}:3000/databases/${database}?access_token=${result.token}`) + .disableTLSCerts() + .ok(() => true)); - unsetAddonConfig(app.id, 'mongodb', callback); - }); - }); - }); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error tearing down mongodb. Status code: ${response.status} message: ${response.body.message}`); + + addonConfigs.unset(app.id, 'mongodb'); } -function backupMongoDb(app, options, callback) { +async function backupMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Backing up mongodb'); - getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); - getAddonConfigByName(app.id, 'mongodb', '%MONGODB_DATABASE', function (error, database) { - if (error) return callback(error); - if (!database) return callback(new BoxError(BoxError.NOT_FOUND, 'Error backing up mongodb. No database')); + const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); + if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error backing up mongodb. No database'); - const url = `https://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`; - pipeRequestToFile(url, dumpPath('mongodb', app.id), callback); - }); - }); + const url = `https://${result.ip}:3000/databases/${database}/backup?access_token=${result.token}`; + await pipeRequestToFile(url, dumpPath('mongodb', app.id)); } -function restoreMongoDb(app, options, callback) { +async function restoreMongoDb(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - - callback = once(callback); // protect from multiple returns with streams debugApp(app, 'restoreMongoDb'); - getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('mongodb', 'CLOUDRON_MONGODB_TOKEN'); - getAddonConfigByName(app.id, 'mongodb', '%MONGODB_DATABASE', function (error, database) { - if (error) return callback(error); - if (!database) return callback(new BoxError(BoxError.NOT_FOUND, 'Error restoring mongodb. No database')); + const database = await addonConfigs.getByName(app.id, 'mongodb', '%MONGODB_DATABASE'); + if (!database) throw new BoxError(BoxError.NOT_FOUND, 'Error restoring mongodb. No database'); - const readStream = fs.createReadStream(dumpPath('mongodb', app.id)); - readStream.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`))); + return new Promise((resolve, reject) => { + reject = once(reject); // protect from multiple returns with streams - const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`)); + const readStream = fs.createReadStream(dumpPath('mongodb', app.id)); + readStream.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring mongodb: ${error.message}`))); - callback(null); - }); + const restoreReq = request.post(`https://${result.ip}:3000/databases/${database}/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { + if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb: ${error.message}`)); + if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring mongodb. Status code: ${response.statusCode} message: ${response.body.message}`)); - readStream.pipe(restoreReq); + resolve(); }); + + readStream.pipe(restoreReq); }); } -function startGraphite(existingInfra, serviceConfig, callback) { +async function startGraphite(existingInfra, serviceConfig) { assert.strictEqual(typeof existingInfra, 'object'); assert.strictEqual(typeof serviceConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); const tag = infra.images.graphite.tag; const memoryLimit = serviceConfig.memoryLimit || 256 * 1024 * 1024; @@ -1828,212 +1637,171 @@ function startGraphite(existingInfra, serviceConfig, callback) { --label isCloudronManaged=true \ --read-only -v /tmp -v /run "${tag}"`; - async.series([ - shell.exec.bind(null, 'stopGraphite', 'docker stop graphite || true'), - shell.exec.bind(null, 'removeGraphite', 'docker rm -f graphite || true'), - (done) => { - if (!upgrading) return done(); - shell.sudo('removeGraphiteDir', [ RMADDONDIR_CMD, 'graphite' ], {}, done); - }, - shell.exec.bind(null, 'startGraphite', cmd) - ], function (error) { - // restart collectd to get the disk stats after graphite starts. currently, there is no way to do graphite health check - if (!error) setTimeout(() => shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK), 60000); + await shell.promises.exec('stopGraphite', 'docker stop graphite || true'); + await shell.promises.exec('removeGraphite', 'docker rm -f graphite || true'); + if (upgrading) await shell.promises.sudo('removeGraphiteDir', [ RMADDONDIR_CMD, 'graphite' ], {}); + await shell.promises.exec('startGraphite', cmd); - callback(error); - }); + // restart collectd to get the disk stats after graphite starts. currently, there is no way to do graphite health check + setTimeout(async () => await safe(shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {})), 60000); } -function setupProxyAuth(app, options, callback) { +async function setupProxyAuth(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Setting up proxyAuth'); const enabled = app.sso && app.manifest.addons && app.manifest.addons.proxyAuth; - if (!enabled) return callback(); + if (!enabled) return; const env = [ { name: 'CLOUDRON_PROXY_AUTH', value: '1' } ]; - setAddonConfig(app.id, 'proxyauth', env, callback); + await addonConfigs.set(app.id, 'proxyauth', env); } -function teardownProxyAuth(app, options, callback) { +async function teardownProxyAuth(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - unsetAddonConfig(app.id, 'proxyauth', callback); + await addonConfigs.unset(app.id, 'proxyauth'); } -function startRedis(existingInfra, callback) { +async function startRedis(existingInfra) { assert.strictEqual(typeof existingInfra, 'object'); - assert.strictEqual(typeof callback, 'function'); const tag = infra.images.redis.tag; const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.redis.tag, tag); - util.callbackify(apps.list)(function (error, allApps) { - if (error) return callback(error); + const allApps = await apps.list(); - async.eachSeries(allApps, function iterator (app, iteratorCallback) { - if (!('redis' in app.manifest.addons)) return iteratorCallback(); // app doesn't use the addon + for (const app of allApps) { + if (!('redis' in app.manifest.addons)) continue; // app doesn't use the addon - const redisName = 'redis-' + app.id; + const redisName = 'redis-' + app.id; - async.series([ - (done) => { - if (!upgrading) return done(); - backupRedis(app, {}, done); - }, - shell.exec.bind(null, 'stopRedis', `docker stop ${redisName} || true`), // redis will backup as part of signal handling - shell.exec.bind(null, 'removeRedis', `docker rm -f ${redisName} || true`), - setupRedis.bind(null, app, app.manifest.addons.redis) // starts the container - ], iteratorCallback); - }, function (error) { - if (error) return callback(error); + if (upgrading) await backupRedis(app, {}); - if (!upgrading) return callback(); + await shell.promises.exec('stopRedis', `docker stop ${redisName} || true`); // redis will backup as part of signal handling + await shell.promises.exec('removeRedis', `docker rm -f ${redisName} || true`); + await setupRedis(app, app.manifest.addons.redis); // starts the container + } - importDatabase('redis', callback); - }); - }); + + if (upgrading) await importDatabase('redis'); } // Ensures that app's addon redis container is running. Can be called when named container already exists/running -function setupRedis(app, options, callback) { +async function setupRedis(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); const redisName = 'redis-' + app.id; - getAddonConfigByName(app.id, 'redis', '%REDIS_PASSWORD', function (error, existingPassword) { - if (error) return callback(error); + const existingPassword = await addonConfigs.getByName(app.id, 'redis', '%REDIS_PASSWORD'); - const redisPassword = options.noPassword ? '' : (existingPassword || hat(4 * 48)); // see box#362 for password length - const redisServiceToken = hat(4 * 48); + const redisPassword = options.noPassword ? '' : (existingPassword || hat(4 * 48)); // see box#362 for password length + const redisServiceToken = hat(4 * 48); - // Compute redis memory limit based on app's memory limit (this is arbitrary) - const memoryLimit = app.servicesConfig['redis'] ? app.servicesConfig['redis'].memoryLimit : APP_SERVICES['redis'].defaultMemoryLimit; - const memory = system.getMemoryAllocation(memoryLimit); + // Compute redis memory limit based on app's memory limit (this is arbitrary) + const memoryLimit = app.servicesConfig['redis'] ? app.servicesConfig['redis'].memoryLimit : APP_SERVICES['redis'].defaultMemoryLimit; + const memory = system.getMemoryAllocation(memoryLimit); - const tag = infra.images.redis.tag; - const label = app.fqdn; - // note that we do not add appId label because this interferes with the stop/start app logic - const cmd = `docker run --restart=always -d --name=${redisName} \ - --hostname ${redisName} \ - --label=location=${label} \ - --net cloudron \ - --net-alias ${redisName} \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag="${redisName}" \ - -m ${memory} \ - --memory-swap ${memoryLimit} \ - --dns 172.18.0.1 \ - --dns-search=. \ - -e CLOUDRON_REDIS_PASSWORD="${redisPassword}" \ - -e CLOUDRON_REDIS_TOKEN="${redisServiceToken}" \ - -v "${paths.PLATFORM_DATA_DIR}/redis/${app.id}:/var/lib/redis" \ - --label isCloudronManaged=true \ - --read-only -v /tmp -v /run ${tag}`; + const tag = infra.images.redis.tag; + const label = app.fqdn; + // note that we do not add appId label because this interferes with the stop/start app logic + const cmd = `docker run --restart=always -d --name=${redisName} \ + --hostname ${redisName} \ + --label=location=${label} \ + --net cloudron \ + --net-alias ${redisName} \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag="${redisName}" \ + -m ${memory} \ + --memory-swap ${memoryLimit} \ + --dns 172.18.0.1 \ + --dns-search=. \ + -e CLOUDRON_REDIS_PASSWORD="${redisPassword}" \ + -e CLOUDRON_REDIS_TOKEN="${redisServiceToken}" \ + -v "${paths.PLATFORM_DATA_DIR}/redis/${app.id}:/var/lib/redis" \ + --label isCloudronManaged=true \ + --read-only -v /tmp -v /run ${tag}`; - const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; + const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; - const env = [ - { name: `${envPrefix}REDIS_URL`, value: 'redis://redisuser:' + redisPassword + '@redis-' + app.id }, - { name: `${envPrefix}REDIS_PASSWORD`, value: redisPassword }, - { name: `${envPrefix}REDIS_HOST`, value: redisName }, - { name: `${envPrefix}REDIS_PORT`, value: '6379' } - ]; + const env = [ + { name: `${envPrefix}REDIS_URL`, value: 'redis://redisuser:' + redisPassword + '@redis-' + app.id }, + { name: `${envPrefix}REDIS_PASSWORD`, value: redisPassword }, + { name: `${envPrefix}REDIS_HOST`, value: redisName }, + { name: `${envPrefix}REDIS_PORT`, value: '6379' } + ]; - async.series([ - (next) => { - docker.inspect(redisName, function (inspectError, result) { // fast-path - if (!inspectError) { - debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`); - return next(); - } - shell.exec('startRedis', cmd, next); - }); - }, - setAddonConfig.bind(null, app.id, 'redis', env), - waitForContainer.bind(null, 'redis-' + app.id, 'CLOUDRON_REDIS_TOKEN') - ], function (error) { - if (error) debug('Error setting up redis: ', error); - callback(error); - }); - }); + const [inspectError, result] = await docker.inspect(redisName); + if (inspectError) { + await shell.promises.exec('startRedis', cmd); + } else { // fast path + debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`); + } + + await addonConfigs.set(app.id, 'redis', env); + await waitForContainer('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); } -function clearRedis(app, options, callback) { +async function clearRedis(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Clearing redis'); - getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); - request.post(`https://${result.ip}:3000/clear?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error clearing redis. Status code: ${response.statusCode} message: ${response.body.message}`)); + const [networkError, response] = await safe(superagent.post(`https://${result.ip}:3000/clear?access_token=${result.token}`) + .disableTLSCerts() + .ok(() => true)); - callback(null); - }); - }); + if (networkError) throw new BoxError(BoxError.ADDONS_ERROR, `Network error clearing redis: ${networkError.message}`); + if (response.status !== 200) throw new BoxError(BoxError.ADDONS_ERROR, `Error clearing redis. Status code: ${response.status} message: ${response.body.message}`); } -function teardownRedis(app, options, callback) { +async function teardownRedis(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); - docker.deleteContainer(`redis-${app.id}`, function (error) { - if (error) return callback(error); + await docker.delContainer(`redis-${app.id}`); - shell.sudo('removeVolume', [ RMADDONDIR_CMD, 'redis', app.id ], {}, function (error) { - if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`)); + const [error] = await safe(shell.promises.sudo('removeVolume', [ RMADDONDIR_CMD, 'redis', app.id ], {})); + if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing redis data: ${error.message}`); - rimraf(path.join(paths.LOG_DIR, `redis-${app.id}`), function (error) { - if (error) debugApp(app, 'cannot cleanup logs:', error); + safe.fs.rmSync(path.join(paths.LOG_DIR, `redis-${app.id}`), { recursive: true, force: true }); + if (safe.error) debugApp(app, 'cannot cleanup logs:', safe.error); - unsetAddonConfig(app.id, 'redis', callback); - }); - }); - }); + await addonConfigs.unset(app.id, 'redis'); } -function backupRedis(app, options, callback) { +async function backupRedis(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Backing up redis'); - getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) { - if (error) return callback(error); + const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); - const url = `https://${result.ip}:3000/backup?access_token=${result.token}`; - pipeRequestToFile(url, dumpPath('redis', app.id), callback); - }); + const url = `https://${result.ip}:3000/backup?access_token=${result.token}`; + await pipeRequestToFile(url, dumpPath('redis', app.id)); } -function restoreRedis(app, options, callback) { +async function restoreRedis(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'Restoring redis'); - callback = once(callback); // protect from multiple returns with streams + const result = await getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN'); - getContainerDetails('redis-' + app.id, 'CLOUDRON_REDIS_TOKEN', function (error, result) { - if (error) return callback(error); + return new Promise((resolve, reject) => { + reject = once(reject); // protect from multiple returns with streams let input; const newDumpLocation = dumpPath('redis', app.id); @@ -2042,149 +1810,112 @@ function restoreRedis(app, options, callback) { } else { // old location of dumps input = fs.createReadStream(path.join(paths.APPS_DATA_DIR, app.id, 'redis/dump.rdb')); } - input.on('error', (error) => callback(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`))); + input.on('error', (error) => reject(new BoxError(BoxError.FS_ERROR, `Error reading input stream when restoring redis: ${error.message}`))); const restoreReq = request.post(`https://${result.ip}:3000/restore?access_token=${result.token}`, { json: true, rejectUnauthorized: false }, function (error, response) { - if (error) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`)); - if (response.statusCode !== 200) return callback(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`)); + if (error) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis: ${error.message}`)); + if (response.statusCode !== 200) return reject(new BoxError(BoxError.ADDONS_ERROR, `Error restoring redis. Status code: ${response.statusCode} message: ${response.body.message}`)); - callback(null); + resolve(); }); input.pipe(restoreReq); }); } -function statusTurn(callback) { - assert.strictEqual(typeof callback, 'function'); +async function statusTurn() { + const [error, container] = await safe(docker.inspect('turn')); + if (error && error.reason === BoxError.NOT_FOUND) return { status: exports.SERVICE_STATUS_STOPPED }; + if (error) throw error; - docker.inspect('turn', function (error, container) { - if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED }); - if (error) return callback(error); + const result = await docker.memoryUsage(container.Id); - docker.memoryUsage(container.Id, function (error, result) { - if (error) return callback(error); - - var tmp = { - status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, - memoryUsed: result.memory_stats.usage, - memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit) - }; - - callback(null, tmp); - }); - }); + return { + status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, + memoryUsed: result.memory_stats.usage, + memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit) + }; } -function statusDocker(callback) { - assert.strictEqual(typeof callback, 'function'); - - docker.ping(function (error) { - callback(null, { status: error ? exports.SERVICE_STATUS_STOPPED: exports.SERVICE_STATUS_ACTIVE }); - }); +async function statusDocker() { + const [error] = await safe(docker.ping()); + return { status: error ? exports.SERVICE_STATUS_STOPPED: exports.SERVICE_STATUS_ACTIVE }; } -function restartDocker(callback) { - assert.strictEqual(typeof callback, 'function'); - - shell.sudo('restartdocker', [ RESTART_SERVICE_CMD, 'docker' ], {}, NOOP_CALLBACK); - - callback(null); +async function restartDocker() { + const [error] = await safe(shell.promises.sudo('restartdocker', [ RESTART_SERVICE_CMD, 'docker' ], {})); + if (error) debug(`restartDocker: error restarting docker. ${error.message}`); } -function statusUnbound(callback) { - assert.strictEqual(typeof callback, 'function'); - - shell.exec('statusUnbound', 'systemctl is-active unbound', function (error) { - callback(null, { status: error ? exports.SERVICE_STATUS_STOPPED : exports.SERVICE_STATUS_ACTIVE }); - }); +async function statusUnbound() { + const [error] = await shell.promises.exec('statusUnbound', 'systemctl is-active unbound'); + return { status: error ? exports.SERVICE_STATUS_STOPPED : exports.SERVICE_STATUS_ACTIVE }; } -function restartUnbound(callback) { - assert.strictEqual(typeof callback, 'function'); - - shell.sudo('restartunbound', [ RESTART_SERVICE_CMD, 'unbound' ], {}, NOOP_CALLBACK); - - callback(null); +async function restartUnbound() { + const [error] = await safe(shell.promises.sudo('restartunbound', [ RESTART_SERVICE_CMD, 'unbound' ], {})); + if (error) debug(`restartDocker: error restarting unbound. ${error.message}`); } -function statusNginx(callback) { - assert.strictEqual(typeof callback, 'function'); - - shell.exec('statusNginx', 'systemctl is-active nginx', function (error) { - callback(null, { status: error ? exports.SERVICE_STATUS_STOPPED : exports.SERVICE_STATUS_ACTIVE }); - }); +async function statusNginx() { + const [error] = await safe(shell.promises.exec('statusNginx', 'systemctl is-active nginx')); + return { status: error ? exports.SERVICE_STATUS_STOPPED : exports.SERVICE_STATUS_ACTIVE }; } -function restartNginx(callback) { - assert.strictEqual(typeof callback, 'function'); - - shell.sudo('restartnginx', [ RESTART_SERVICE_CMD, 'nginx' ], {}, NOOP_CALLBACK); - - callback(null); +async function restartNginx() { + const [error] = await safe(shell.promises.sudo('restartnginx', [ RESTART_SERVICE_CMD, 'nginx' ], {})); + if (error) debug(`restartNginx: error restarting unbound. ${error.message}`); } -function statusSftp(callback) { - assert.strictEqual(typeof callback, 'function'); +async function statusSftp() { + const [error, container] = await safe(docker.inspect('sftp')); + if (error && error.reason === BoxError.NOT_FOUND) return { status: exports.SERVICE_STATUS_STOPPED }; + if (error) throw error; - docker.inspect('sftp', function (error, container) { - if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED }); - if (error) return callback(error); + const result = await docker.memoryUsage('sftp'); - docker.memoryUsage('sftp', function (error, result) { - if (error) return callback(error); - - var tmp = { - status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, - memoryUsed: result.memory_stats.usage, - memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit) - }; - - callback(null, tmp); - }); - }); + return { + status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, + memoryUsed: result.memory_stats.usage, + memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit) + }; } -function statusGraphite(callback) { - assert.strictEqual(typeof callback, 'function'); +async function statusGraphite() { + const [error, container] = await safe(docker.inspect('graphite')); + if (error && error.reason === BoxError.NOT_FOUND) return { status: exports.SERVICE_STATUS_STOPPED }; + if (error) throw error; - docker.inspect('graphite', function (error, container) { - if (error && error.reason === BoxError.NOT_FOUND) return callback(null, { status: exports.SERVICE_STATUS_STOPPED }); - if (error) return callback(error); + const [networkError, response] = await safe(superagent.get('http://127.0.0.1:8417/graphite-web/dashboard') + .timeout(20000) + .ok(() => true)); - request.get('http://127.0.0.1:8417/graphite-web/dashboard', { json: true, timeout: 20000 }, function (error, response) { - if (error) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${error.message}` }); - if (response.statusCode !== 200) return callback(null, { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.statusCode} message: ${response.body.message}` }); + if (networkError) return { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite: ${networkError.message}` }; + if (response.status !== 200) return { status: exports.SERVICE_STATUS_STARTING, error: `Error waiting for graphite. Status code: ${response.status} message: ${response.body.message}` }; - docker.memoryUsage('graphite', function (error, result) { - if (error) return callback(error); + const result = await docker.memoryUsage('graphite'); - var tmp = { - status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, - memoryUsed: result.memory_stats.usage, - memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit) - }; - - callback(null, tmp); - }); - }); - }); + return { + status: container.State.Running ? exports.SERVICE_STATUS_ACTIVE : exports.SERVICE_STATUS_STOPPED, + memoryUsed: result.memory_stats.usage, + memoryPercent: parseInt(100 * result.memory_stats.usage / result.memory_stats.limit) + }; } -function restartGraphite(callback) { - assert.strictEqual(typeof callback, 'function'); +async function restartGraphite() { + await docker.restartContainer('graphite'); - docker.restartContainer('graphite', callback); - - setTimeout(() => shell.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {}, NOOP_CALLBACK), 60000); + setTimeout(async () => { + const [error] = await safe(shell.promises.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {})); + if (error) debug(`restartGraphite: error restarting collected. ${error.message}`); + }, 60000); } -function teardownOauth(app, options, callback) { +async function teardownOauth(app, options) { assert.strictEqual(typeof app, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debugApp(app, 'teardownOauth'); - unsetAddonConfig(app.id, 'oauth', callback); + await addonConfigs.unset(app.id, 'oauth'); } diff --git a/src/settings.js b/src/settings.js index 13074fce2..087348892 100644 --- a/src/settings.js +++ b/src/settings.js @@ -499,8 +499,7 @@ async function setRegistryConfig(registryConfig) { docker.injectPrivateFields(registryConfig, currentConfig); - const dockerTestRegistryConfig = util.promisify(docker.testRegistryConfig); - await dockerTestRegistryConfig(registryConfig); + await docker.testRegistryConfig(registryConfig); await set(exports.REGISTRY_CONFIG_KEY, JSON.stringify(registryConfig)); diff --git a/src/sftp.js b/src/sftp.js index 00f26418c..b102ae665 100644 --- a/src/sftp.js +++ b/src/sftp.js @@ -9,7 +9,6 @@ exports = module.exports = { const apps = require('./apps.js'), assert = require('assert'), - async = require('async'), BoxError = require('./boxerror.js'), debug = require('debug')('box:sftp'), hat = require('./hat.js'), @@ -18,13 +17,11 @@ const apps = require('./apps.js'), safe = require('safetydance'), shell = require('./shell.js'), system = require('./system.js'), - util = require('util'), volumes = require('./volumes.js'); -function rebuild(serviceConfig, options, callback) { +async function rebuild(serviceConfig, options) { assert.strictEqual(typeof serviceConfig, 'object'); assert.strictEqual(typeof options, 'object'); - assert.strictEqual(typeof callback, 'function'); debug('rebuilding container'); @@ -36,78 +33,71 @@ function rebuild(serviceConfig, options, callback) { let dataDirs = []; const stat = safe.fs.lstatSync(paths.APPS_DATA_DIR); - if (!stat) return callback(new BoxError(BoxError.FS_ERROR, safe.error)); + if (!stat) throw new BoxError(BoxError.FS_ERROR, safe.error); const resolvedAppDataDir = stat.isSymbolicLink() ? safe.fs.readlinkSync(paths.APPS_DATA_DIR) : paths.APPS_DATA_DIR; dataDirs.push({ hostDir: resolvedAppDataDir, mountDir: '/mnt/appsdata' }); - util.callbackify(apps.list)(async function (error, result) { - if (error) return callback(error); + const result = await apps.list(); - result.forEach(function (app) { - if (!app.manifest.addons['localstorage'] || !app.dataDir) return; + result.forEach(function (app) { + if (!app.manifest.addons['localstorage'] || !app.dataDir) return; - const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/mnt/${app.id}`; - if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user - // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail - debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); - return; - } + const hostDir = apps.getDataDir(app, app.dataDir), mountDir = `/mnt/${app.id}`; + if (!safe.fs.existsSync(hostDir)) { // this can fail if external mount does not have permissions for yellowtent user + // do not create host path when cloudron is restoring. this will then create dir with root perms making restore logic fail + debug(`Ignoring app data dir ${hostDir} for ${app.id} since it does not exist`); + return; + } - dataDirs.push({ hostDir, mountDir }); - }); - - let allVolumes; - [error, allVolumes] = await safe(volumes.list()); - if (error) return callback(error); - - dataDirs.push({ hostDir: '/mnt/volumes', mountDir: '/mnt/volumes' }); - - allVolumes.forEach(function (volume) { - if (volume.hostPath.startsWith('/mnt/volumes/')) return; - - if (!safe.fs.existsSync(volume.hostPath)) { - debug(`Ignoring volume host path ${volume.hostPath} since it does not exist`); - return; - } - - dataDirs.push({ hostDir: volume.hostPath, mountDir: `/mnt/${volume.id}` }); - }); - - const mounts = dataDirs.map(function (v) { return `-v "${v.hostDir}:${v.mountDir}"`; }).join(' '); - const cmd = `docker run --restart=always -d --name="sftp" \ - --hostname sftp \ - --net cloudron \ - --net-alias sftp \ - --log-driver syslog \ - --log-opt syslog-address=udp://127.0.0.1:2514 \ - --log-opt syslog-format=rfc5424 \ - --log-opt tag=sftp \ - -m ${memory} \ - --memory-swap ${memoryLimit} \ - --dns 172.18.0.1 \ - --dns-search=. \ - -p 222:22 \ - ${mounts} \ - -e CLOUDRON_SFTP_TOKEN="${cloudronToken}" \ - -v "${paths.SFTP_KEYS_DIR}:/etc/ssh:ro" \ - --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; - - // ignore error if container not found (and fail later) so that this code works across restarts - async.series([ - shell.exec.bind(null, 'stopSftp', 'docker stop sftp || true'), - shell.exec.bind(null, 'removeSftp', 'docker rm -f sftp || true'), - shell.exec.bind(null, 'startSftp', cmd) - ], callback); + dataDirs.push({ hostDir, mountDir }); }); + + let allVolumes = await volumes.list(); + + dataDirs.push({ hostDir: '/mnt/volumes', mountDir: '/mnt/volumes' }); + + allVolumes.forEach(function (volume) { + if (volume.hostPath.startsWith('/mnt/volumes/')) return; + + if (!safe.fs.existsSync(volume.hostPath)) { + debug(`Ignoring volume host path ${volume.hostPath} since it does not exist`); + return; + } + + dataDirs.push({ hostDir: volume.hostPath, mountDir: `/mnt/${volume.id}` }); + }); + + const mounts = dataDirs.map(function (v) { return `-v "${v.hostDir}:${v.mountDir}"`; }).join(' '); + const cmd = `docker run --restart=always -d --name="sftp" \ + --hostname sftp \ + --net cloudron \ + --net-alias sftp \ + --log-driver syslog \ + --log-opt syslog-address=udp://127.0.0.1:2514 \ + --log-opt syslog-format=rfc5424 \ + --log-opt tag=sftp \ + -m ${memory} \ + --memory-swap ${memoryLimit} \ + --dns 172.18.0.1 \ + --dns-search=. \ + -p 222:22 \ + ${mounts} \ + -e CLOUDRON_SFTP_TOKEN="${cloudronToken}" \ + -v "${paths.SFTP_KEYS_DIR}:/etc/ssh:ro" \ + --label isCloudronManaged=true \ + --read-only -v /tmp -v /run "${tag}"`; + + // ignore error if container not found (and fail later) so that this code works across restarts + await shell.promises.exec('stopSftp', 'docker stop sftp || true'); + await shell.promises.exec('removeSftp', 'docker rm -f sftp || true'); + await shell.promises.exec('startSftp', cmd); } -function start(existingInfra, serviceConfig, callback) { +async function start(existingInfra, serviceConfig) { assert.strictEqual(typeof existingInfra, 'object'); assert.strictEqual(typeof serviceConfig, 'object'); - assert.strictEqual(typeof callback, 'function'); - rebuild(serviceConfig, { force: true }, callback); // force rebuild when infra changed + await rebuild(serviceConfig, { force: true }); // force rebuild when infra changed } diff --git a/src/system.js b/src/system.js index 30a9b9004..cbcb684e0 100644 --- a/src/system.js +++ b/src/system.js @@ -9,7 +9,6 @@ exports = module.exports = { const apps = require('./apps.js'), assert = require('assert'), - async = require('async'), BoxError = require('./boxerror.js'), debug = require('debug')('box:disks'), df = require('@sindresorhus/df'), @@ -21,8 +20,6 @@ const apps = require('./apps.js'), settings = require('./settings.js'), volumes = require('./volumes.js'); -const dfAsync = async.asyncify(df), dfFileAsync = async.asyncify(df.file); - async function getVolumeDisks(appsDataDisk) { assert.strictEqual(typeof appsDataDisk, 'string'); @@ -55,7 +52,7 @@ async function getAppDisks(appsDataDisk) { return appDisks; } -async function getBackupDisk() { +async function getBackupsFilesystem() { const backupConfig = await settings.getBackupConfig(); if (backupConfig.provider !== 'filesystem') return null; @@ -64,82 +61,67 @@ async function getBackupDisk() { return result.filesystem; } -function getDisks(callback) { - assert.strictEqual(typeof callback, 'function'); +async function getDisks() { + const info = await docker.info(); - docker.info(function (error, info) { - if (error) return callback(error); + let [error, allDisks] = await safe(df()); + if (error) throw new BoxError(BoxError.FS_ERROR, error); - async.series([ - dfAsync, - dfFileAsync.bind(null, paths.BOX_DATA_DIR), - dfFileAsync.bind(null, paths.PLATFORM_DATA_DIR), - dfFileAsync.bind(null, paths.APPS_DATA_DIR), - dfFileAsync.bind(null, info.DockerRootDir), - getBackupDisk, - ], async function (error, values) { - if (error) return callback(new BoxError(BoxError.FS_ERROR, error)); + // filter by ext4 and then sort to make sure root disk is first + const ext4Disks = allDisks.filter((r) => r.type === 'ext4').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint)); - // filter by ext4 and then sort to make sure root disk is first - const ext4Disks = values[0].filter((r) => r.type === 'ext4').sort((a, b) => a.mountpoint.localeCompare(b.mountpoint)); + const diskInfos = []; + for (const p of [ paths.BOX_DATA_DIR, paths.PLATFORM_DATA_DIR, paths.APPS_DATA_DIR, info.DockerRootDir ]) { + const [dfError, diskInfo] = await safe(df.file(p)); + if (dfError) throw new BoxError(BoxError.FS_ERROR, dfError); + diskInfos.push(diskInfo); + } - const disks = { - disks: ext4Disks, // root disk is first. { filesystem, type, size, used, avialable, capacity, mountpoint } - boxDataDisk: values[1].filesystem, - mailDataDisk: values[1].filesystem, - platformDataDisk: values[2].filesystem, - appsDataDisk: values[3].filesystem, - dockerDataDisk: values[4].filesystem, - backupsDisk: values[5], - apps: {}, // filled below - volumes: {} // filled below - }; + const backupsFilesystem = await getBackupsFilesystem(); - [error, disks.apps] = await safe(getAppDisks(disks.appsDataDisk)); - if (error) return callback(error); + const result = { + disks: ext4Disks, // root disk is first. { filesystem, type, size, used, avialable, capacity, mountpoint } + boxDataDisk: diskInfos[0].filesystem, + mailDataDisk: diskInfos[0].filesystem, + platformDataDisk: diskInfos[1].filesystem, + appsDataDisk: diskInfos[2].filesystem, + dockerDataDisk: diskInfos[3].filesystem, + backupsDisk: backupsFilesystem, + apps: {}, // filled below + volumes: {} // filled below + }; - [error, disks.volumes] = await safe(getVolumeDisks(disks.appsDataDisk)); - if (error) return callback(error); + result.apps = await getAppDisks(result.appsDataDisk); + result.volumes = await getVolumeDisks(result.appsDataDisk); - callback(null, disks); - }); - }); + return result; } -function checkDiskSpace(callback) { - assert.strictEqual(typeof callback, 'function'); - +async function checkDiskSpace() { debug('checkDiskSpace: checking disk space'); - getDisks(async function (error, disks) { - if (error) { - debug('checkDiskSpace: error getting disks %s', error.message); - return callback(); + const disks = await getDisks(); + + let markdownMessage = ''; + + disks.disks.forEach(function (entry) { + // ignore other filesystems but where box, app and platform data is + if (entry.filesystem !== disks.boxDataDisk + && entry.filesystem !== disks.platformDataDisk + && entry.filesystem !== disks.appsDataDisk + && entry.filesystem !== disks.backupsDisk + && entry.filesystem !== disks.dockerDataDisk) return false; + + if (entry.available <= (1.25 * 1024 * 1024 * 1024)) { // 1.5G + markdownMessage += `* ${entry.filesystem} is at ${entry.capacity*100}% capacity.\n`; } - - let markdownMessage = ''; - - disks.disks.forEach(function (entry) { - // ignore other filesystems but where box, app and platform data is - if (entry.filesystem !== disks.boxDataDisk - && entry.filesystem !== disks.platformDataDisk - && entry.filesystem !== disks.appsDataDisk - && entry.filesystem !== disks.backupsDisk - && entry.filesystem !== disks.dockerDataDisk) return false; - - if (entry.available <= (1.25 * 1024 * 1024 * 1024)) { // 1.5G - markdownMessage += `* ${entry.filesystem} is at ${entry.capacity*100}% capacity.\n`; - } - }); - - debug(`checkDiskSpace: disk space checked. out of space: ${markdownMessage || 'no'}`); - - if (markdownMessage) markdownMessage = `One or more file systems are running out of space. Please increase the disk size at the earliest.\n\n${markdownMessage}`; - - await notifications.alert(notifications.ALERT_DISK_SPACE, 'Server is running out of disk space', markdownMessage); - - callback(); }); + + debug(`checkDiskSpace: disk space checked. out of space: ${markdownMessage || 'no'}`); + + if (markdownMessage) markdownMessage = `One or more file systems are running out of space. Please increase the disk size at the earliest.\n\n${markdownMessage}`; + + await notifications.alert(notifications.ALERT_DISK_SPACE, 'Server is running out of disk space', markdownMessage); } function getSwapSize() { @@ -149,13 +131,11 @@ function getSwapSize() { return swap; } -function getMemory(callback) { - assert.strictEqual(typeof callback, 'function'); - - callback(null, { +async function getMemory() { + return { memory: os.totalmem(), swap: getSwapSize() - }); + }; } function getMemoryAllocation(limit) { diff --git a/src/test/system-test.js b/src/test/system-test.js index 298228d28..d12f86d85 100644 --- a/src/test/system-test.js +++ b/src/test/system-test.js @@ -16,30 +16,19 @@ describe('System', function () { before(setup); after(cleanup); - it('can get disks', function (done) { - system.getDisks(function (error, disks) { - expect(!error).to.be.ok(); - expect(disks).to.be.ok(); - done(); - }); + it('can get disks', async function () { + const disks = await system.getDisks(); + expect(disks).to.be.ok(); }); - it('can check for disk space', function (done) { - system.checkDiskSpace(function (error) { - expect(!error).to.be.ok(); - done(); - }); + it('can check for disk space', async function () { + await system.checkDiskSpace(); }); - it('can get memory', function (done) { - system.getMemory(function (error, memory) { - expect(!error).to.be.ok(); - - expect(memory.memory).to.be.a('number'); - expect(memory.swap).to.be.a('number'); - - done(); - }); + it('can get memory', async function () { + const memory = await system.getMemory(); + expect(memory.memory).to.be.a('number'); + expect(memory.swap).to.be.a('number'); }); });