cloudron-box/src/platform.js

'use strict';

exports = module.exports = {
    start: start,
    stop: stop,

    createMailConfig: createMailConfig
};

var apps = require('./apps.js'),
    assert = require('assert'),
    async = require('async'),
    config = require('./config.js'),
    certificates = require('./certificates.js'),
    debug = require('debug')('box:platform'),
    fs = require('fs'),
    hat = require('hat'),
    infra = require('./infra_version.js'),
    nginx = require('./nginx.js'),
    os = require('os'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    semver = require('semver'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    subdomains = require('./subdomains.js'),
    taskmanager = require('./taskmanager.js'),
    user = require('./user.js'),
    util = require('util'),
    _ = require('underscore');

var gPlatformReadyTimer = null;

var NOOP_CALLBACK = function (error) { if (error) debug(error); };

function start(callback) {
    assert.strictEqual(typeof callback, 'function');

    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback();

    debug('initializing addon infrastructure');

    // restart mail container if any of these keys change
    settings.events.on(settings.MAIL_CONFIG_KEY, function () { startMail(NOOP_CALLBACK); });
    settings.events.on(settings.MAIL_RELAY_KEY, function () { startMail(NOOP_CALLBACK); });

    certificates.events.on(certificates.EVENT_CERT_CHANGED, function (domain) {
        if (domain === '*.' + config.fqdn() || domain === config.adminFqdn()) startMail(NOOP_CALLBACK);
    });

    var existingInfra = { version: 'none' };
    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
        if (!existingInfra) existingInfra = { version: 'corrupt' };
    }

    // short-circuit for the restart case
    if (_.isEqual(infra, existingInfra)) {
        debug('platform is uptodate at version %s', infra.version);
        emitPlatformReady();
        return callback();
    }

    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);

    async.series([
        stopContainers.bind(null, existingInfra),
        startAddons.bind(null, existingInfra),
        removeOldImages,
        startApps.bind(null, existingInfra),
        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra))
    ], function (error) {
        if (error) return callback(error);

        emitPlatformReady();

        callback();
    });
}

function stop(callback) {
    clearTimeout(gPlatformReadyTimer);
    gPlatformReadyTimer = null;
    exports.events = null;
    taskmanager.pauseTasks(callback);
}

function emitPlatformReady() {
    // give some time for the platform to "settle". For example, mysql might still be initing the
    // database dir and we cannot call service scripts until that's done.
    // TODO: make this smarter to not wait for 15secs for the crash-restart case
    gPlatformReadyTimer = setTimeout(function () {
        debug('emitting platform ready');
        gPlatformReadyTimer = null;
        taskmanager.resumeTasks();
    }, 15000);
}

function removeOldImages(callback) {
    debug('removing old addon images');

    for (var imageName in infra.images) {
        if (imageName === 'redis') continue; // see #223
        var image = infra.images[imageName];
        debug('cleaning up images of %j', image);
        var cmd = 'docker images "%s" | tail -n +2 | awk \'{ print $1 ":" $2 }\' | grep -v "%s" | xargs --no-run-if-empty docker rmi';
        shell.execSync('removeOldImagesSync', util.format(cmd, image.repo, image.tag));
    }

    callback();
}

function stopContainers(existingInfra, callback) {
    // TODO: be nice and stop addons cleanly (example, shutdown commands)

    // always stop addons to restart them on any infra change, regardless of minor or major update
    if (existingInfra.version !== infra.version) {
        debug('stopping all containers for infra upgrade');
        shell.execSync('stopContainers', 'docker ps -qa | xargs --no-run-if-empty docker rm -f');
    } else {
        assert(typeof infra.images, 'object');
        var changedAddons = [ ];
        for (var imageName in infra.images) {
            if (imageName === 'redis') continue; // see #223
            if (infra.images[imageName].tag !== existingInfra.images[imageName].tag) changedAddons.push(imageName);
        }

        debug('stopping addons for incremental infra update: %j', changedAddons);
        // ignore error if container not found (and fail later) so that this code works across restarts
        shell.execSync('stopContainers', 'docker rm -f ' + changedAddons.join(' ') + ' || true');
    }

    callback();
}

function startGraphite(callback) {
    const tag = infra.images.graphite.tag;
    const dataDir = paths.PLATFORM_DATA_DIR;

    const cmd = `docker run --restart=always -d --name="graphite" \
                --net cloudron \
                --net-alias graphite \
                -m 75m \
                --memory-swap 150m \
                --dns 172.18.0.1 \
                --dns-search=. \
                -p 127.0.0.1:2003:2003 \
                -p 127.0.0.1:2004:2004 \
                -p 127.0.0.1:8000:8000 \
                -v "${dataDir}/graphite:/app/data" \
                --read-only -v /tmp -v /run "${tag}"`;

    shell.execSync('startGraphite', cmd);

    callback();
}

function startMysql(callback) {
    const tag = infra.images.mysql.tag;
    const dataDir = paths.PLATFORM_DATA_DIR;
    const rootPassword = hat(8 * 128);
    const memoryLimit = (1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 256;

    if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mysql_vars.sh',
            'MYSQL_ROOT_PASSWORD=' + rootPassword +'\nMYSQL_ROOT_HOST=172.18.0.1', 'utf8')) {
        return callback(new Error('Could not create mysql var file:' + safe.error.message));
    }

    const cmd = `docker run --restart=always -d --name="mysql" \
                --net cloudron \
                --net-alias mysql \
                -m ${memoryLimit}m \
                --memory-swap ${memoryLimit * 2}m \
                --dns 172.18.0.1 \
                --dns-search=. \
                -v "${dataDir}/mysql:/var/lib/mysql" \
                -v "${dataDir}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
                --read-only -v /tmp -v /run "${tag}"`;

    shell.execSync('startMysql', cmd);

    setTimeout(callback, 5000);
}

function startPostgresql(callback) {
    const tag = infra.images.postgresql.tag;
    const dataDir = paths.PLATFORM_DATA_DIR;
    const rootPassword = hat(8 * 128);
    const memoryLimit = (1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 256;

    if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/postgresql_vars.sh', 'POSTGRESQL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
        return callback(new Error('Could not create postgresql var file:' + safe.error.message));
    }

    const cmd = `docker run --restart=always -d --name="postgresql" \
                --net cloudron \
                --net-alias postgresql \
                -m ${memoryLimit}m \
                --memory-swap ${memoryLimit * 2}m \
                --dns 172.18.0.1 \
                --dns-search=. \
                -v "${dataDir}/postgresql:/var/lib/postgresql" \
                -v "${dataDir}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
                --read-only -v /tmp -v /run "${tag}"`;

    shell.execSync('startPostgresql', cmd);

    setTimeout(callback, 5000);
}

function startMongodb(callback) {
    const tag = infra.images.mongodb.tag;
    const dataDir = paths.PLATFORM_DATA_DIR;
    const rootPassword = hat(8 * 128);
    const memoryLimit = (1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 200;

    if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mongodb_vars.sh', 'MONGODB_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
        return callback(new Error('Could not create mongodb var file:' + safe.error.message));
    }

    const cmd = `docker run --restart=always -d --name="mongodb" \
                --net cloudron \
                --net-alias mongodb \
                -m ${memoryLimit}m \
                --memory-swap ${memoryLimit * 2}m \
                --dns 172.18.0.1 \
                --dns-search=. \
                -v "${dataDir}/mongodb:/var/lib/mongodb" \
                -v "${dataDir}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
                --read-only -v /tmp -v /run "${tag}"`;

    shell.execSync('startMongodb', cmd);

    setTimeout(callback, 5000);
}

function createMailConfig(callback) {
    assert.strictEqual(typeof callback, 'function');

    const fqdn = config.fqdn();
    const mailFqdn = config.adminFqdn();
    const alertsFrom = 'no-reply@' + config.fqdn();

    debug('createMailConfig: generating mail config');

    user.getOwner(function (error, owner) {
        var alertsTo = config.provider() === 'caas' ? [ 'support@cloudron.io' ] : [ ];
        alertsTo.concat(error ? [] : owner.email).join(','); // owner may not exist yet

        settings.getAll(function (error, result) {
            if (error) return callback(error);

            var catchAll = result[settings.CATCH_ALL_ADDRESS_KEY].join(',');
            var mailFromValidation = result[settings.MAIL_FROM_VALIDATION_KEY];

            if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mail/mail.ini',
                `mail_domain=${fqdn}\nmail_server_name=${mailFqdn}\nalerts_from=${alertsFrom}\nalerts_to=${alertsTo}\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n`, 'utf8')) {
                return callback(new Error('Could not create mail var file:' + safe.error.message));
            }

            var relay = result[settings.MAIL_RELAY_KEY];

            const enabled = relay.provider !== 'cloudron-smtp' ? true : false,
                  host = relay.host || '',
                  port = relay.port || 25,
                  username = relay.username || '',
                  password = relay.password || '';

            if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mail/smtp_forward.ini',
                `enable_outbound=${enabled}\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=plain\nauth_user=${username}\nauth_pass=${password}`, 'utf8')) {
                return callback(new Error('Could not create mail var file:' + safe.error.message));
            }

            callback();
        });
    });
}

function startMail(callback) {
    // mail (note: 2525 is hardcoded in mail container and app use this port)
    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
    // MAIL_DOMAIN is the domain for which this server is relaying mails
    // mail container uses /app/data for backed up data and /run for restart-able data

    const tag = infra.images.mail.tag;
    const dataDir = paths.PLATFORM_DATA_DIR;
    const memoryLimit = Math.max((1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 128, 256);

    // admin and mail share the same certificate
    certificates.getAdminCertificate(function (error, cert, key) {
        if (error) return callback(error);

        if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mail/tls_cert.pem', cert)) return callback(new Error('Could not create cert file:' + safe.error.message));
        if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mail/tls_key.pem', key))  return callback(new Error('Could not create key file:' + safe.error.message));

        settings.getMailConfig(function (error, mailConfig) {
            if (error) return callback(error);

            shell.execSync('startMail', 'docker rm -f mail || true');

            createMailConfig(function (error) {
                if (error) return callback(error);

                var ports = mailConfig.enabled ? '-p 587:2525 -p 993:9993 -p 4190:4190 -p 25:2525' : '';

                const cmd = `docker run --restart=always -d --name="mail" \
                            --net cloudron \
                            --net-alias mail \
                            -m ${memoryLimit}m \
                            --memory-swap ${memoryLimit * 2}m \
                            --dns 172.18.0.1 \
                            --dns-search=. \
                            --env ENABLE_MDA=${mailConfig.enabled} \
                            -v "${dataDir}/mail:/app/data" \
                            -v "${dataDir}/addons/mail:/etc/mail" \
                            ${ports} \
                            --read-only -v /run -v /tmp ${tag}`;

                shell.execSync('startMail', cmd);

                if (!mailConfig.enabled || process.env.BOX_ENV === 'test') return callback();

                // Add MX and DMARC record. Note that DMARC policy depends on DKIM signing and thus works
                // only if we use our internal mail server.
                var records = [
                    { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] },
                    { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] }
                ];

                async.mapSeries(records, function (record, iteratorCallback) {
                    subdomains.upsert(record.subdomain, record.type, record.values, iteratorCallback);
                }, callback);
            });
        });
    });
}

function startAddons(existingInfra, callback) {
    var startFuncs = [ ];

    // always start addons on any infra change, regardless of minor or major update
    if (existingInfra.version !== infra.version) {
        debug('startAddons: no existing infra or infra upgrade. starting all addons');
        startFuncs.push(startGraphite, startMysql, startPostgresql, startMongodb, startMail);
    } else {
        assert.strictEqual(typeof existingInfra.images, 'object');

        if (infra.images.graphite.tag !== existingInfra.images.graphite.tag) startFuncs.push(startGraphite);
        if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) startFuncs.push(startMysql);
        if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) startFuncs.push(startPostgresql);
        if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) startFuncs.push(startMongodb);
        if (infra.images.mail.tag !== existingInfra.images.mail.tag) startFuncs.push(startMail);

        debug('startAddons: existing infra. incremental addon create %j', startFuncs.map(function (f) { return f.name; }));
    }

    async.series(startFuncs, callback);
}

function startApps(existingInfra, callback) {
    // Infra version change strategy:
    // * no existing version - restore apps
    // * major versions - restore apps
    // * minor versions - reconfigure apps

    if (existingInfra.version === infra.version) {
        debug('startApp: apps are already uptodate');
        callback();
    } else if (existingInfra.version === 'none' || !semver.valid(existingInfra.version) || semver.major(existingInfra.version) !== semver.major(infra.version)) {
        debug('startApps: restoring installed apps');
        apps.restoreInstalledApps(callback);
    } else {
        debug('startApps: reconfiguring installed apps');
        nginx.removeAppConfigs(); // should we change the cert location, nginx will not start
        apps.configureInstalledApps(callback);
    }
}