diff --git a/CHANGES b/CHANGES index a6b32ad65..3f2ebd254 100644 --- a/CHANGES +++ b/CHANGES @@ -2351,4 +2351,5 @@ * eventlog: add service rebuild/restart/configure events * upcloud: add object storage integration * Each app can now have a custom crontab +* services: add recovery mode diff --git a/src/mail.js b/src/mail.js index e58d8fd08..37c102b8a 100644 --- a/src/mail.js +++ b/src/mail.js @@ -695,8 +695,10 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) { const allowInbound = await createMailConfig(mailFqdn, mailDomain); const ports = allowInbound ? '-p 587:2587 -p 993:9993 -p 4190:4190 -p 25:2587 -p 465:2465' : ''; + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; - const cmd = `docker run --restart=always -d --name="mail" \ + const runCmd = `docker run --restart=always -d --name="mail" \ --net cloudron \ --net-alias mail \ --log-driver syslog \ @@ -713,9 +715,9 @@ async function configureMail(mailFqdn, mailDomain, serviceConfig) { -v "${paths.PLATFORM_DATA_DIR}/addons/mail:/etc/mail" \ ${ports} \ --label isCloudronManaged=true \ - --read-only -v /run -v /tmp ${tag}`; + ${readOnly} -v /run -v /tmp ${tag} ${cmd}`; - await shell.promises.exec('startMail', cmd); + await shell.promises.exec('startMail', runCmd); } async function getMailAuth() { diff --git a/src/routes/services.js b/src/routes/services.js index 5a344c7e4..d29a79c29 100644 --- a/src/routes/services.js +++ b/src/routes/services.js @@ -40,9 +40,11 @@ async function configure(req, res, next) { assert.strictEqual(typeof req.params.service, 'string'); if (typeof req.body.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit must be a number')); + if ('recoveryMode' in req.body && typeof req.body.recoveryMode !== 'boolean') return next(new HttpError(400, 'recoveryMode must be boolean')); const data = { - memoryLimit: req.body.memoryLimit + memoryLimit: req.body.memoryLimit, + recoveryMode: req.body.recoveryMode || false }; const [error] = await safe(services.configureService(req.params.service, data, AuditSource.fromRequest(req))); diff --git a/src/services.js b/src/services.js index 15530b4e6..a5539e7b5 100644 --- a/src/services.js +++ b/src/services.js @@ -384,6 +384,7 @@ async function configureService(id, data, auditSource) { assert.strictEqual(typeof auditSource, 'object'); const [name, instance ] = id.split(':'); + let needsRebuild = false; if (instance) { if (!APP_SERVICES[name]) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); @@ -392,21 +393,27 @@ async function configureService(id, data, auditSource) { if (!app) throw new BoxError(BoxError.NOT_FOUND, 'App not found'); const servicesConfig = app.servicesConfig; + needsRebuild = servicesConfig[name]?.recoveryMode != data.recoveryMode; servicesConfig[name] = data; await apps.update(instance, { servicesConfig }); - - await applyServiceConfig(id, data); } else if (SERVICES[name]) { const servicesConfig = await settings.getServicesConfig(); + needsRebuild = servicesConfig[name]?.recoveryMode != data.recoveryMode; // intentional != since 'debug' may or may not be there servicesConfig[name] = data; await settings.setServicesConfig(servicesConfig); - await applyServiceConfig(id, data); } else { throw new BoxError(BoxError.NOT_FOUND, 'No such service'); } + // do this in background + if (needsRebuild) { + safe(rebuildService(id, auditSource), { debug }); + } else { + safe(applyMemoryLimit(id), { debug }); + } + await eventlog.add(eventlog.ACTION_SERVICE_CONFIGURE, auditSource, { id, data }); } @@ -516,8 +523,9 @@ async function rebuildService(id, auditSource) { // nothing to rebuild for now. } - // TODO: missing redis container is not created + safe(applyMemoryLimit(id), { debug }); // do this in background. ok to fail + // TODO: missing redis container is not created await eventlog.add(eventlog.ACTION_SERVICE_REBUILD, auditSource, { id }); } @@ -744,12 +752,12 @@ async function exportDatabase(addon) { await shell.promises.sudo(`exportDatabase - removeAddonDir${addon}`, [ RMADDONDIR_CMD, addon ], {}); // ready to start afresh } -async function applyServiceConfig(id, serviceConfig) { +async function applyMemoryLimit(id) { assert.strictEqual(typeof id, 'string'); - assert.strictEqual(typeof serviceConfig, 'object'); const [name, instance] = id.split(':'); let containerName, memoryLimit; + const serviceConfig = await getServiceConfig(id); if (instance) { if (!APP_SERVICES[name]) throw new BoxError(BoxError.NOT_FOUND, 'Service not found'); @@ -807,10 +815,8 @@ async function startServices(existingInfra) { } // we always start db containers with unlimited memory. we then scale them down per configuration - const servicesConfig = await settings.getServicesConfig(); for (const id of [ 'mysql', 'postgresql', 'mongodb' ]) { - const serviceConfig = servicesConfig[id] || {}; - safe(applyServiceConfig(id, serviceConfig), { debug }); // no waiting. and it's ok if applying service configs fails + safe(applyMemoryLimit(id), { debug }); // no waiting. and it's ok if applying service configs fails } } @@ -911,8 +917,11 @@ async function startTurn(existingInfra) { const turnSecret = await blobs.get(blobs.ADDON_TURN_SECRET); if (!turnSecret) throw new BoxError(BoxError.ADDONS_ERROR, 'Turn secret is missing'); + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + // this exports 3478/tcp, 5349/tls and 50000-51000/udp. note that this runs on the host network! - const cmd = `docker run --restart=always -d --name="turn" \ + const runCmd = `docker run --restart=always -d --name="turn" \ --hostname turn \ --net host \ --log-driver syslog \ @@ -926,11 +935,11 @@ async function startTurn(existingInfra) { -e CLOUDRON_TURN_SECRET="${turnSecret}" \ -e CLOUDRON_REALM="${realm}" \ --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`; await shell.promises.exec('stopTurn', 'docker stop turn || true'); await shell.promises.exec('removeTurn', 'docker rm -f turn || true'); - await shell.promises.exec('startTurn', cmd); + await shell.promises.exec('startTurn', runCmd); } async function teardownTurn(app, options) { @@ -1107,8 +1116,12 @@ async function startMysql(existingInfra) { await exportDatabase('mysql'); } + const serviceConfig = await getServiceConfig('mysql'); + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + // memory options are applied dynamically. import requires all the memory we can get - const cmd = `docker run --restart=always -d --name="mysql" \ + const runCmd = `docker run --restart=always -d --name="mysql" \ --hostname mysql \ --net cloudron \ --net-alias mysql \ @@ -1124,11 +1137,11 @@ async function startMysql(existingInfra) { -v "${dataDir}/mysql:/var/lib/mysql" \ --label isCloudronManaged=true \ --cap-add SYS_NICE \ - --read-only -v /tmp -v /run "${tag}"`; + ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`; await shell.promises.exec('stopMysql', 'docker stop mysql || true'); await shell.promises.exec('removeMysql', 'docker rm -f mysql || true'); - await shell.promises.exec('startMysql', cmd); + await shell.promises.exec('startMysql', runCmd); await waitForContainer('mysql', 'CLOUDRON_MYSQL_TOKEN'); @@ -1308,8 +1321,12 @@ async function startPostgresql(existingInfra) { await exportDatabase('postgresql'); } + const serviceConfig = await getServiceConfig('postgresql'); + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + // memory options are applied dynamically. import requires all the memory we can get - const cmd = `docker run --restart=always -d --name="postgresql" \ + const runCmd = `docker run --restart=always -d --name="postgresql" \ --hostname postgresql \ --net cloudron \ --net-alias postgresql \ @@ -1324,11 +1341,11 @@ async function startPostgresql(existingInfra) { -e CLOUDRON_POSTGRESQL_TOKEN="${cloudronToken}" \ -v "${dataDir}/postgresql:/var/lib/postgresql" \ --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`; await shell.promises.exec('stopPostgresql', 'docker stop postgresql || true'); await shell.promises.exec('removePostgresql', 'docker rm -f postgresql || true'); - await shell.promises.exec('startPostgresql', cmd); + await shell.promises.exec('startPostgresql', runCmd); await waitForContainer('postgresql', 'CLOUDRON_POSTGRESQL_TOKEN'); if (upgrading) await importDatabase('postgresql'); @@ -1466,8 +1483,12 @@ async function startMongodb(existingInfra) { await exportDatabase('mongodb'); } + const serviceConfig = await getServiceConfig('mongodb'); + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + // memory options are applied dynamically. import requires all the memory we can get - const cmd = `docker run --restart=always -d --name="mongodb" \ + const runCmd = `docker run --restart=always -d --name="mongodb" \ --hostname mongodb \ --net cloudron \ --net-alias mongodb \ @@ -1481,11 +1502,11 @@ async function startMongodb(existingInfra) { -e CLOUDRON_MONGODB_TOKEN="${cloudronToken}" \ -v "${dataDir}/mongodb:/var/lib/mongodb" \ --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`; await shell.promises.exec('stopMongodb', 'docker stop mongodb || true'); await shell.promises.exec('removeMongodb', 'docker rm -f mongodb || true'); - await shell.promises.exec('startMongodb', cmd); + await shell.promises.exec('startMongodb', runCmd); await waitForContainer('mongodb', 'CLOUDRON_MONGODB_TOKEN'); if (upgrading) await importDatabase('mongodb'); @@ -1628,7 +1649,10 @@ async function startGraphite(existingInfra) { if (upgrading) debug('startGraphite: graphite will be upgraded'); - const cmd = `docker run --restart=always -d --name="graphite" \ + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + + const runCmd = `docker run --restart=always -d --name="graphite" \ --hostname graphite \ --net cloudron \ --net-alias graphite \ @@ -1645,12 +1669,12 @@ async function startGraphite(existingInfra) { -p 127.0.0.1:8417:8000 \ -v "${paths.PLATFORM_DATA_DIR}/graphite:/var/lib/graphite" \ --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`; await shell.promises.exec('stopGraphite', 'docker stop graphite || true'); await shell.promises.exec('removeGraphite', 'docker rm -f graphite || true'); if (upgrading) await shell.promises.sudo('removeGraphiteDir', [ RMADDONDIR_CMD, 'graphite' ], {}); - await shell.promises.exec('startGraphite', cmd); + await shell.promises.exec('startGraphite', runCmd); // restart collectd to get the disk stats after graphite starts. currently, there is no way to do graphite health check setTimeout(async () => await safe(shell.promises.sudo('restartcollectd', [ RESTART_SERVICE_CMD, 'collectd' ], {})), 60000); @@ -1697,7 +1721,6 @@ async function startRedis(existingInfra) { await setupRedis(app, app.manifest.addons.redis); // starts the container } - if (upgrading) await importDatabase('redis'); } @@ -1714,13 +1737,17 @@ async function setupRedis(app, options) { const redisServiceToken = hat(4 * 48); // Compute redis memory limit based on app's memory limit (this is arbitrary) - const memoryLimit = app.servicesConfig['redis'] ? app.servicesConfig['redis'].memoryLimit : APP_SERVICES['redis'].defaultMemoryLimit; + const memoryLimit = app.servicesConfig['redis']?.memoryLimit || APP_SERVICES['redis'].defaultMemoryLimit; const memory = system.getMemoryAllocation(memoryLimit); + const recoveryMode = app.servicesConfig['redis']?.recoveryMode || false; + const readOnly = !recoveryMode ? '--read-only' : ''; + const cmd = recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + const tag = infra.images.redis.tag; const label = app.fqdn; // note that we do not add appId label because this interferes with the stop/start app logic - const cmd = `docker run --restart=always -d --name=${redisName} \ + const runCmd = `docker run --restart=always -d --name=${redisName} \ --hostname ${redisName} \ --label=location=${label} \ --net cloudron \ @@ -1737,7 +1764,7 @@ async function setupRedis(app, options) { -e CLOUDRON_REDIS_TOKEN="${redisServiceToken}" \ -v "${paths.PLATFORM_DATA_DIR}/redis/${app.id}:/var/lib/redis" \ --label isCloudronManaged=true \ - --read-only -v /tmp -v /run ${tag}`; + ${readOnly} -v /tmp -v /run ${tag} ${cmd}`; const envPrefix = app.manifest.manifestVersion <= 1 ? '' : 'CLOUDRON_'; @@ -1750,7 +1777,7 @@ async function setupRedis(app, options) { const [inspectError, result] = await safe(docker.inspect(redisName)); if (inspectError) { - await shell.promises.exec('startRedis', cmd); + await shell.promises.exec('startRedis', runCmd); } else { // fast path debug(`Re-using existing redis container with state: ${JSON.stringify(result.State)}`); } diff --git a/src/sftp.js b/src/sftp.js index 9cbde5817..b1ba40643 100644 --- a/src/sftp.js +++ b/src/sftp.js @@ -70,8 +70,11 @@ async function start(existingInfra) { if (!resolvedMailDataDir) throw new BoxError(BoxError.FS_ERROR, `Could not resolve mail data dir: ${safe.error.message}`); dataDirs.push({ hostDir: resolvedMailDataDir, mountDir: '/mnt/maildata' }); + const readOnly = !serviceConfig.recoveryMode ? '--read-only' : ''; + const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : ''; + const mounts = dataDirs.map(v => `-v "${v.hostDir}:${v.mountDir}"`).join(' '); - const cmd = `docker run --restart=always -d --name="sftp" \ + const runCmd = `docker run --restart=always -d --name="sftp" \ --hostname sftp \ --net cloudron \ --net-alias sftp \ @@ -88,10 +91,10 @@ async function start(existingInfra) { -e CLOUDRON_SFTP_TOKEN="${cloudronToken}" \ -v "${paths.SFTP_KEYS_DIR}:/etc/ssh:ro" \ --label isCloudronManaged=true \ - --read-only -v /tmp -v /run "${tag}"`; + ${readOnly} -v /tmp -v /run "${tag}" ${cmd}`; // ignore error if container not found (and fail later) so that this code works across restarts await shell.promises.exec('stopSftp', 'docker stop sftp || true'); await shell.promises.exec('removeSftp', 'docker rm -f sftp || true'); - await shell.promises.exec('startSftp', cmd); + await shell.promises.exec('startSftp', runCmd); }