'use strict'; exports = module.exports = { start, stopAllTasks, getStatus }; const apps = require('./apps.js'), assert = require('assert'), AuditSource = require('./auditsource.js'), BoxError = require('./boxerror.js'), constants = require('./constants.js'), debug = require('debug')('box:platform'), delay = require('./delay.js'), fs = require('fs'), infra = require('./infra_version.js'), locker = require('./locker.js'), paths = require('./paths.js'), reverseProxy = require('./reverseproxy.js'), safe = require('safetydance'), services = require('./services.js'), shell = require('./shell.js'), tasks = require('./tasks.js'), volumes = require('./volumes.js'), _ = require('underscore'); let gStatusMessage = 'Initializing'; function getStatus() { return { message: gStatusMessage }; } async function start(options) { if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return; debug('initializing platform'); let existingInfra = { version: 'none' }; if (fs.existsSync(paths.INFRA_VERSION_FILE)) { existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8')); if (!existingInfra) existingInfra = { version: 'corrupt' }; } // short-circuit for the restart case if (_.isEqual(infra, existingInfra)) { debug('platform is uptodate at version %s', infra.version); await onPlatformReady(false /* !infraChanged */); return; } debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version); const error = locker.lock(locker.OP_PLATFORM_START); if (error) throw error; for (let attempt = 0; attempt < 5; attempt++) { try { if (existingInfra.version !== infra.version) { gStatusMessage = 'Removing containers for upgrade'; await removeAllContainers(); await createDockerNetwork(); } if (existingInfra.version === 'none') await volumes.mountAll(); // when restoring, mount all volumes await markApps(existingInfra, options); // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored gStatusMessage = 'Starting services, this can take a while'; await services.startServices(existingInfra); await fs.promises.writeFile(paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4)); break; } catch (error) { // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck // LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED'); debug(`Failed to start services. retry=${retry} (attempt ${attempt}): ${error.message}`); if (!retry) throw error; // refuse to start await delay(10000); } } locker.unlock(locker.OP_PLATFORM_START); await onPlatformReady(true /* infraChanged */); } async function stopAllTasks() { await tasks.stopAllTasks(); } async function onPlatformReady(infraChanged) { debug(`onPlatformReady: platform is ready. infra changed: ${infraChanged}`); gStatusMessage = 'Ready'; if (infraChanged) await safe(pruneInfraImages(), { debug }); // ignore error await apps.schedulePendingTasks(AuditSource.PLATFORM); } async function pruneInfraImages() { debug('pruneInfraImages: checking existing images'); // cannot blindly remove all unused images since redis image may not be used const images = infra.baseImages.concat(Object.keys(infra.images).map(function (addon) { return infra.images[addon]; })); for (const image of images) { const output = safe.child_process.execSync(`docker images --digests ${image.repo} --format "{{.ID}} {{.Repository}}:{{.Tag}}@{{.Digest}}"`, { encoding: 'utf8' }); if (output === null) { debug(`Failed to list images of ${image}. %o`, safe.error); throw safe.error; } const lines = output.trim().split('\n'); for (const line of lines) { if (!line) continue; const parts = line.split(' '); // [ ID, Repo:Tag@Digest ] const normalizedTag = parts[1].replace('registry.ipv6.docker.com/', '').replace('registry-1.docker.io/', '').replace('registry.docker.com', ''); if (image.tag === normalizedTag) continue; // keep debug(`pruneInfraImages: removing unused image of ${image.repo}: tag: ${parts[1]} id: ${parts[0]}`); let result = safe.child_process.execSync(`docker rmi ${parts[1].replace(':', '')}`, { encoding: 'utf8' }); // the none tag has to be removed if (result === null) debug(`Error removing image ${parts[0]}: ${safe.error.mesage}`); } } } async function createDockerNetwork() { debug('createDockerNetwork: recreating docker network'); await shell.promises.exec('createDockerNetwork', 'docker network rm cloudron || true'); // the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA await shell.promises.exec('createDockerNetwork', `docker network create --subnet=${constants.DOCKER_IPv4_SUBNET} --ip-range=${constants.DOCKER_IPv4_RANGE} --gateway ${constants.DOCKER_IPv4_GATEWAY} --ipv6 --subnet=fd00:c107:d509::/64 cloudron`); } async function removeAllContainers() { debug('removeAllContainers: removing all containers for infra upgrade'); await shell.promises.exec('removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop'); await shell.promises.exec('removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f'); } async function markApps(existingInfra, options) { assert.strictEqual(typeof existingInfra, 'object'); assert.strictEqual(typeof options, 'object'); if (existingInfra.version === 'none') { // cloudron is being restored from backup debug('markApps: restoring installed apps'); await apps.restoreInstalledApps(options, AuditSource.PLATFORM); } else if (existingInfra.version !== infra.version) { debug('markApps: reconfiguring installed apps'); reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start await apps.configureInstalledApps(AuditSource.PLATFORM); } else { let changedAddons = []; if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) changedAddons.push('mysql'); if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) changedAddons.push('postgresql'); if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) changedAddons.push('mongodb'); if (infra.images.redis.tag !== existingInfra.images.redis.tag) changedAddons.push('redis'); if (changedAddons.length) { // restart apps if docker image changes since the IP changes and any "persistent" connections fail debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`); await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM); } else { debug('markApps: apps are already uptodate'); } } }