cloudron-box/src/platform.js

'use strict';

exports = module.exports = {
    start,
    stopAllTasks,

    getStatus
};

const apps = require('./apps.js'),
    assert = require('assert'),
    AuditSource = require('./auditsource.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    debug = require('debug')('box:platform'),
    delay = require('./delay.js'),
    fs = require('fs'),
    infra = require('./infra_version.js'),
    locker = require('./locker.js'),
    paths = require('./paths.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    shell = require('./shell.js'),
    tasks = require('./tasks.js'),
    volumes = require('./volumes.js'),
    _ = require('underscore');

let gStatusMessage = 'Initializing';

function getStatus() {
    return { message: gStatusMessage };
}

async function start(options) {
    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return;

    debug('initializing platform');

    let existingInfra = { version: 'none' };
    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
        if (!existingInfra) existingInfra = { version: 'corrupt' };
    }

    // short-circuit for the restart case
    if (_.isEqual(infra, existingInfra)) {
        debug('platform is uptodate at version %s', infra.version);
        await onPlatformReady(false /* !infraChanged */);
        return;
    }

    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);

    const error = locker.lock(locker.OP_PLATFORM_START);
    if (error) throw error;

    for (let attempt = 0; attempt < 5; attempt++) {
        try {
            if (existingInfra.version !== infra.version) {
                gStatusMessage = 'Removing containers for upgrade';
                await removeAllContainers();
                await createDockerNetwork();
            }
            if (existingInfra.version === 'none') await volumes.mountAll(); // when restoring, mount all volumes
            await markApps(existingInfra, options); // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
            gStatusMessage = 'Starting services, this can take a while';
            await services.startServices(existingInfra);
            await fs.promises.writeFile(paths.INFRA_VERSION_FILE, JSON.stringify(infra, null, 4));
            break;
        } catch (error) {
            // for some reason, mysql arbitrary restarts making startup tasks fail. this makes the box update stuck
            // LOST is when existing connection breaks. REFUSED is when new connection cannot connect at all
            const retry = error.reason === BoxError.DATABASE_ERROR && (error.code === 'PROTOCOL_CONNECTION_LOST' || error.code === 'ECONNREFUSED');
            debug(`Failed to start services. retry=${retry} (attempt ${attempt}): ${error.message}`);
            if (!retry) throw error; // refuse to start
            await delay(10000);
        }
    }

    locker.unlock(locker.OP_PLATFORM_START);

    await onPlatformReady(true /* infraChanged */);
}

async function stopAllTasks() {
    await tasks.stopAllTasks();
}

async function onPlatformReady(infraChanged) {
    debug(`onPlatformReady: platform is ready. infra changed: ${infraChanged}`);
    gStatusMessage = 'Ready';

    if (infraChanged) await safe(pruneInfraImages(), { debug }); // ignore error

    await apps.schedulePendingTasks(AuditSource.PLATFORM);
}

async function pruneInfraImages() {
    debug('pruneInfraImages: checking existing images');

    // cannot blindly remove all unused images since redis image may not be used
    const images = infra.baseImages.concat(Object.keys(infra.images).map(function (addon) { return infra.images[addon]; }));

    for (const image of images) {
        const output = safe.child_process.execSync(`docker images --digests ${image.repo} --format "{{.ID}} {{.Repository}}:{{.Tag}}@{{.Digest}}"`, { encoding: 'utf8' });
        if (output === null) {
            debug(`Failed to list images of ${image}`, safe.error);
            throw safe.error;
        }

        const lines = output.trim().split('\n');
        for (const line of lines) {
            if (!line) continue;
            const parts = line.split(' '); // [ ID, Repo:Tag@Digest ]
            const normalizedTag = parts[1].replace('registry.ipv6.docker.com/', '').replace('registry-1.docker.io/', '');

            if (image.tag === normalizedTag) continue; // keep
            debug(`pruneInfraImages: removing unused image of ${image.repo}: tag: ${parts[1]} id: ${parts[0]}`);

            let result = safe.child_process.execSync(`docker rmi ${parts[1].replace(':<none>', '')}`, { encoding: 'utf8' }); // the none tag has to be removed
            if (result === null) debug(`Error removing image ${parts[0]}: ${safe.error.mesage}`);
        }
    }
}

async function createDockerNetwork() {
    debug('createDockerNetwork: recreating docker network');

    await shell.promises.exec('createDockerNetwork', 'docker network rm cloudron || true');
    // the --ipv6 option will work even in ipv6 is disabled. fd00 is IPv6 ULA
    await shell.promises.exec('createDockerNetwork', `docker network create --subnet=${constants.DOCKER_IPv4_SUBNET} --ip-range=${constants.DOCKER_IPv4_RANGE} --gateway ${constants.DOCKER_IPv4_GATEWAY} --ipv6 --subnet=fd00:c107:d509::/64 cloudron`);
}

async function removeAllContainers() {
    debug('removeAllContainers: removing all containers for infra upgrade');

    await shell.promises.exec('removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop');
    await shell.promises.exec('removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f');
}

async function markApps(existingInfra, options) {
    assert.strictEqual(typeof existingInfra, 'object');
    assert.strictEqual(typeof options, 'object');

    if (existingInfra.version === 'none') { // cloudron is being restored from backup
        debug('markApps: restoring installed apps');
        await apps.restoreInstalledApps(options, AuditSource.PLATFORM);
    } else if (existingInfra.version !== infra.version) {
        debug('markApps: reconfiguring installed apps');
        reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start
        await apps.configureInstalledApps(AuditSource.PLATFORM);
    } else {
        let changedAddons = [];
        if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) changedAddons.push('mysql');
        if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) changedAddons.push('postgresql');
        if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) changedAddons.push('mongodb');
        if (infra.images.redis.tag !== existingInfra.images.redis.tag) changedAddons.push('redis');

        if (changedAddons.length) {
            // restart apps if docker image changes since the IP changes and any "persistent" connections fail
            debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
            await apps.restartAppsUsingAddons(changedAddons, AuditSource.PLATFORM);
        } else {
            debug('markApps: apps are already uptodate');
        }
    }
}