cloudron-box/src/platform.js

'use strict';

exports = module.exports = {
    initialize,
    uninitialize,

    onActivated,
    onDashboardLocationSet,
    onDashboardLocationChanged,
    onMailServerLocationChanged,

    getStatus
};

const apps = require('./apps.js'),
    assert = require('assert'),
    AuditSource = require('./auditsource.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    cron = require('./cron.js'),
    dashboard = require('./dashboard.js'),
    database = require('./database.js'),
    debug = require('debug')('box:platform'),
    docker = require('./docker.js'),
    dockerProxy = require('./dockerproxy.js'),
    fs = require('fs'),
    infra = require('./infra_version.js'),
    locker = require('./locker.js'),
    oidc = require('./oidc.js'),
    paths = require('./paths.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    shell = require('./shell.js')('platform'),
    tasks = require('./tasks.js'),
    timers = require('timers/promises'),
    updater = require('./updater.js'),
    users = require('./users.js'),
    volumes = require('./volumes.js'),
    _ = require('underscore');

let gStatusMessage = 'Initializing';

function getStatus() {
    return { message: gStatusMessage };
}

async function pruneInfraImages() {
    debug('pruneInfraImages: checking existing images');

    // cannot blindly remove all unused images since redis image may not be used
    const imageNames = Object.keys(infra.images).map(addon => infra.images[addon]);
    const [error, output] = await safe(shell.exec('docker images --digests --format "{{.ID}} {{.Repository}} {{.Tag}} {{.Digest}}"', { shell: '/bin/bash' }));
    if (error) {
        debug(`Failed to list images ${error.message}`);
        throw error;
    }
    const lines = output.trim().split('\n');

    for (const imageName of imageNames) {
        const parsedTag = docker.parseImageName(imageName);

        for (const line of lines) {
            if (!line) continue;
            const [, repo, tag, digest] = line.split(' '); // [ ID, Repo, Tag, Digest ]
            const normalizedRepo = repo.replace('registry.ipv6.docker.com/', '').replace('registry-1.docker.io/', '').replace('registry.docker.com/', '');
            if (!parsedTag.repository.endsWith(normalizedRepo)) continue; // some other repo
            if (imageName === `${repo}:${tag}@${digest}`) continue; // the image we want to keep

            const imageIdToPrune = tag === '<none>' ? `${repo}@${digest}` : `${repo}:${tag}`; // untagged, use digest
            console.log(`pruneInfraImages: removing unused image of ${imageName}: ${imageIdToPrune}`);

            const [error] = await safe(shell.spawn('docker', [ 'rmi', imageIdToPrune ], {}));
            if (error) console.log(`Error removing image ${imageIdToPrune}: ${error.mesage}`);
        }
    }
}

async function pruneVolumes() {
    debug('pruneVolumes: remove all unused local volumes');

    const [error] = await safe(shell.spawn('docker', [ 'volume', 'prune', '--all', '--force' ], {}));
    if (error) console.log(`Error pruning volumes: ${error.mesage}`);
}

async function createDockerNetwork() {
    debug('createDockerNetwork: recreating docker network');

    await shell.spawn('docker', ['network', 'rm', '-f', 'cloudron'], {});
    // the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA
    await shell.spawn('docker', ['network', 'create', `--subnet=${constants.DOCKER_IPv4_SUBNET}`, `--ip-range=${constants.DOCKER_IPv4_RANGE}`,
        `--gateway ${constants.DOCKER_IPv4_GATEWAY}`, '--ipv6', '--subnet=fd00:c107:d509::/64', 'cloudron'], {});
}

async function removeAllContainers() {
    debug('removeAllContainers: removing all containers for infra upgrade');

    await shell.exec('docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop', { shell: '/bin/bash' });
    await shell.exec('docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f', { shell: '/bin/bash' });
}

async function markApps(existingInfra, restoreOptions) {
    assert.strictEqual(typeof existingInfra, 'object');
    assert.strictEqual(typeof restoreOptions, 'object');

    if (existingInfra.version === 'none') { // cloudron is being restored from backup
        debug('markApps: restoring apps');
        await apps.restoreApps(await apps.list(), restoreOptions, AuditSource.PLATFORM);
    } else if (existingInfra.version !== infra.version) {
        debug('markApps: reconfiguring apps');
        reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start
        await apps.configureApps(await apps.list(), { scheduleNow: false }, AuditSource.PLATFORM); // we will schedule it when infra is ready
    } else {
        const changedAddons = [];
        if (infra.images.mysql !== existingInfra.images.mysql) changedAddons.push('mysql');
        if (infra.images.postgresql !== existingInfra.images.postgresql) changedAddons.push('postgresql');
        if (infra.images.mongodb !== existingInfra.images.mongodb) changedAddons.push('mongodb');
        if (infra.images.redis !== existingInfra.images.redis) changedAddons.push('redis');

        if (changedAddons.length) {
            // restart apps if docker image changes since the IP changes and any "persistent" connections fail
            debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
            await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM);
        } else {
            debug('markApps: apps are already uptodate');
        }
    }
}

async function onInfraReady(infraChanged) {
    debug(`onInfraReady: platform is ready. infra changed: ${infraChanged}`);
    gStatusMessage = 'Ready';

    if (infraChanged) {
        await safe(pruneInfraImages(), { debug }); // ignore error
        await safe(pruneVolumes(), { debug }); // ignore error
    }

    await apps.schedulePendingTasks(AuditSource.PLATFORM);
}

async function startInfra(restoreOptions) {
    assert.strictEqual(typeof restoreOptions, 'object');

    if (constants.TEST && !process.env.TEST_CREATE_INFRA) return;

    debug('startInfra: checking infrastructure');

    let existingInfra = { version: 'none' };
    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
        if (!existingInfra) existingInfra = { version: 'corrupt' };
    }

    // short-circuit for the restart case
    if (_.isEqual(infra, existingInfra)) {
        debug('startInfra: infra is uptodate at version %s', infra.version);
        await onInfraReady(false /* !infraChanged */);
        return;
    }

    debug(`startInfra: updating infrastructure from ${existingInfra.version} to ${infra.version}`);

    const error = locker.lock(locker.OP_INFRA_START);
    if (error) throw error;

    for (let attempt = 0; attempt < 5; attempt++) {
        try {
            if (existingInfra.version !== infra.version) {
                gStatusMessage = 'Removing containers for upgrade';
                await removeAllContainers();
                await createDockerNetwork();
            }
            if (existingInfra.version === 'none') await volumes.mountAll(); // when restoring, mount all volumes
            await markApps(existingInfra, restoreOptions); // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
            gStatusMessage = 'Starting services, this can take a while';
            await services.startServices(existingInfra);
            await fs.promises.writeFile(paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4));
            break;
        } catch (error) {
            // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck
            // LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all
            const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED');
            debug(`startInfra: Failed to start services. retry=${retry} (attempt ${attempt}): ${error.message}`);
            if (!retry) throw error; // refuse to start
            await timers.setTimeout(10000);
        }
    }

    locker.unlock(locker.OP_INFRA_START);

    await onInfraReady(true /* infraChanged */);
}

async function initialize() {
    debug('initialize: start platform');

    await database.initialize();
    await tasks.stopAllTasks();

    // always generate webadmin config since we have no versioning mechanism for the ejs
    const dashboardLocation = await dashboard.getLocation();
    if (dashboardLocation.domain) await onDashboardLocationSet(dashboardLocation.subdomain, dashboardLocation.domain);

    // configure nginx to be reachable by IP when not activated. for the moment, the IP based redirect exists even after domain is setup
    // just in case user forgot or some network error happenned in the middle (then browser refresh takes you to activation page)
    // we remove the config as a simple security measure to not expose IP <-> domain
    const activated = await users.isActivated();
    if (!activated) {
        debug('start: not activated. generating IP based redirection config');
        await safe(reverseProxy.writeDefaultConfig({ activated: false }), { debug }); // ok to fail if no disk space
    }

    await updater.notifyUpdate();

    if (await users.isActivated()) safe(onActivated({ skipDnsSetup: false }), { debug }); // run in background
}

async function uninitialize() {
    debug('uninitializing platform');

    if (await users.isActivated()) await onDeactivated();

    await cron.stopJobs();
    await dockerProxy.stop();
    await tasks.stopAllTasks();
    await database.uninitialize();
}

async function onActivated(restoreOptions) {
    assert.strictEqual(typeof restoreOptions, 'object');

    debug('onActivated: starting post activation services');

    // Starting the infra after a user is available means:
    // 1. mail bounces can now be sent to the cloudron owner
    // 2. the restore code path can run without sudo (since mail/ is non-root)
    await startInfra(restoreOptions);
    await cron.startJobs();
    await dockerProxy.start(); // this relies on the 'cloudron' docker network interface to be available

    // disable responding to api calls via IP to not leak domain info. this is carefully placed as the last item, so it buys
    // the UI some time to query the dashboard domain in the restore code path
    if (!constants.TEST) await timers.setTimeout(30000);
    await reverseProxy.writeDefaultConfig({ activated :true });

    debug('onActivated: finished');
}

async function onDeactivated() {
    debug('onDeactivated: stopping post activation services');

    await cron.stopJobs();
    await dockerProxy.stop();
    await oidc.stop();
}

async function onDashboardLocationSet(subdomain, domain) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');

    await safe(reverseProxy.writeDashboardConfig(subdomain, domain), { debug }); // ok to fail if no disk space
    await oidc.stop();
    await oidc.start();
}

async function onDashboardLocationChanged(auditSource) {
    assert.strictEqual(typeof auditSource, 'object');

    // mark all apps to be reconfigured, all have ExtraHosts injected
    const [, installedApps] = await safe(apps.list());
    await safe(apps.configureApps(installedApps, { scheduleNow: true }, auditSource), { debug });

    await safe(services.rebuildService('turn', auditSource), { debug }); // to update the realm variable
}

async function onMailServerLocationChanged(auditSource) {
    assert.strictEqual(typeof auditSource, 'object');

    // mark apps using email addon to be reconfigured
    const [, installedApps] = await safe(apps.list());
    await safe(apps.configureApps(installedApps.filter((a) => !!a.manifest.addons?.email), { scheduleNow: true }, auditSource), { debug });
}