cloudron-box/src/reverseproxy.js

'use strict';

exports = module.exports = {
    setAppCertificate,
    setFallbackCertificate,

    generateFallbackCertificateSync,

    validateCertificate,

    getCertificatePath,
    ensureCertificate,

    checkCerts,

    // the 'configure' ensure a certificate and generate nginx config
    configureApp,
    unconfigureApp,

    // these only generate nginx config
    writeDefaultConfig,
    writeDashboardConfig,
    writeAppConfig,

    removeAppConfigs,
    restoreFallbackCertificates,

    // exported for testing
    _getAcmeApi: getAcmeApi
};

const acme2 = require('./acme2.js'),
    apps = require('./apps.js'),
    assert = require('assert'),
    async = require('async'),
    blobs = require('./blobs.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    crypto = require('crypto'),
    debug = require('debug')('box:reverseproxy'),
    domains = require('./domains.js'),
    ejs = require('ejs'),
    eventlog = require('./eventlog.js'),
    fs = require('fs'),
    mail = require('./mail.js'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    rimraf = require('rimraf'),
    safe = require('safetydance'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    sysinfo = require('./sysinfo.js'),
    users = require('./users.js'),
    util = require('util'),
    _ = require('underscore');

const NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/nginxconfig.ejs', { encoding: 'utf8' });
const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh');

function nginxLocation(s) {
    if (!s.startsWith('!')) return s;

    let re = s.replace(/[\^$\\.*+?()[\]{}|]/g, '\\$&'); // https://github.com/es-shims/regexp.escape/blob/master/implementation.js

    return `~ ^(?!(${re.slice(1)}))`; // negative regex assertion - https://stackoverflow.com/questions/16302897/nginx-location-not-equal-to-regex
}

function getAcmeApi(domainObject, callback) {
    assert.strictEqual(typeof domainObject, 'object');
    assert.strictEqual(typeof callback, 'function');

    const api = acme2;

    let options = { prod: false, performHttpAuthorization: false, wildcard: false, email: '' };
    options.prod = domainObject.tlsConfig.provider.match(/.*-prod/) !== null; // matches 'le-prod' or 'letsencrypt-prod'
    options.performHttpAuthorization = domainObject.provider.match(/noop|manual|wildcard/) !== null;
    options.wildcard = !!domainObject.tlsConfig.wildcard;

    // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
    // we cannot use admin@fqdn because the user might not have set it up.
    // we simply update the account with the latest email we have each time when getting letsencrypt certs
    // https://github.com/ietf-wg-acme/acme/issues/30
    users.getOwner(function (error, owner) {
        options.email = error ? 'webmaster@cloudron.io' : owner.email; // can error if not activated yet

        const blobGet = util.callbackify(blobs.get);
        blobGet(blobs.ACME_ACCOUNT_KEY, function (error, accountKeyPem) {
            if (error) return callback(error);
            if (!accountKeyPem) return callback(new BoxError(BoxError.NOT_FOUND, 'acme account key not found'));

            options.accountKeyPem = accountKeyPem;

            callback(null, api, options);
        });
    });
}

function isExpiringSync(certFilePath, hours) {
    assert.strictEqual(typeof certFilePath, 'string');
    assert.strictEqual(typeof hours, 'number');

    if (!fs.existsSync(certFilePath)) return 2; // not found

    const result = safe.child_process.spawnSync('/usr/bin/openssl', [ 'x509', '-checkend', String(60 * 60 * hours), '-in', certFilePath ]);

    if (!result) return 3; // some error

    debug('isExpiringSync: %s %s %s', certFilePath, result.stdout.toString('utf8').trim(), result.status);

    return result.status === 1; // 1 - expired 0 - not expired
}

// We used to check for the must-staple in the cert using openssl x509 -text -noout -in ${certFilePath} | grep -q status_request
// however, we cannot set the must-staple because first request to nginx fails because of it's OCSP caching behavior
function hasOCSPUriSync(certFilePath) {
    const result = safe.child_process.execSync(`openssl x509 -in ${certFilePath} -noout -ocsp_uri`, { encoding: 'utf8' });
    return result && result.length > 0; // no error and has uri
}

// checks if the certificate matches the options provided by user (like wildcard, le-staging etc)
function providerMatchesSync(domainObject, certFilePath, apiOptions) {
    assert.strictEqual(typeof domainObject, 'object');
    assert.strictEqual(typeof certFilePath, 'string');
    assert.strictEqual(typeof apiOptions, 'object');

    if (!fs.existsSync(certFilePath)) return false; // not found

    const subjectAndIssuer = safe.child_process.execSync(`/usr/bin/openssl x509 -noout -subject -issuer -in "${certFilePath}"`, { encoding: 'utf8' });
    if (!subjectAndIssuer) return false; // something bad happenned

    const subject = subjectAndIssuer.match(/^subject=(.*)$/m)[1];
    const domain = subject.substr(subject.indexOf('=') + 1).trim(); // subject can be /CN=, CN=, CN = and other forms
    const issuer = subjectAndIssuer.match(/^issuer=(.*)$/m)[1];
    const isWildcardCert = domain.includes('*');
    const isLetsEncryptProd = issuer.includes('Let\'s Encrypt') && !issuer.includes('STAGING');

    const issuerMismatch = (apiOptions.prod && !isLetsEncryptProd) || (!apiOptions.prod && isLetsEncryptProd);
    // bare domain is not part of wildcard SAN
    const wildcardMismatch = (domain !== domainObject.domain) && (apiOptions.wildcard && !isWildcardCert) || (!apiOptions.wildcard && isWildcardCert);

    const mismatch = issuerMismatch || wildcardMismatch;

    debug(`providerMatchesSync: ${certFilePath} subject=${subject} domain=${domain} issuer=${issuer} `
        + `wildcard=${isWildcardCert}/${apiOptions.wildcard} prod=${isLetsEncryptProd}/${apiOptions.prod} `
        + `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`);

    return !mismatch;
}

// note: https://tools.ietf.org/html/rfc4346#section-7.4.2 (certificate_list) requires that the
// servers certificate appears first (and not the intermediate cert)
function validateCertificate(location, domainObject, certificate) {
    assert.strictEqual(typeof location, 'string');
    assert.strictEqual(typeof domainObject, 'object');
    assert(certificate && typeof certificate, 'object');

    const cert = certificate.cert, key = certificate.key;

    // check for empty cert and key strings
    if (!cert && key) return new BoxError(BoxError.BAD_FIELD, 'missing cert', { field: 'cert' });
    if (cert && !key) return new BoxError(BoxError.BAD_FIELD, 'missing key', { field: 'key' });

    // -checkhost checks for SAN or CN exclusively. SAN takes precedence and if present, ignores the CN.
    const fqdn = domains.fqdn(location, domainObject);

    let result = safe.child_process.execSync(`openssl x509 -noout -checkhost "${fqdn}"`, { encoding: 'utf8', input: cert });
    if (result === null) return new BoxError(BoxError.BAD_FIELD, 'Unable to get certificate subject:' + safe.error.message, { field: 'cert' });

    if (result.indexOf('does match certificate') === -1) return new BoxError(BoxError.BAD_FIELD, `Certificate is not valid for this domain. Expecting ${fqdn}`, { field: 'cert' });

    // check if public key in the cert and private key matches. pkey below works for RSA and ECDSA keys
    const pubKeyFromCert = safe.child_process.execSync('openssl x509 -noout -pubkey', { encoding: 'utf8', input: cert });
    if (pubKeyFromCert === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from cert: ${safe.error.message}`, { field: 'cert' });

    const pubKeyFromKey = safe.child_process.execSync('openssl pkey -pubout', { encoding: 'utf8', input: key });
    if (pubKeyFromKey === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from private key: ${safe.error.message}`, { field: 'cert' });

    if (pubKeyFromCert !== pubKeyFromKey) return new BoxError(BoxError.BAD_FIELD, 'Public key does not match the certificate.', { field: 'cert' });

    // check expiration
    result = safe.child_process.execSync('openssl x509 -checkend 0', { encoding: 'utf8', input: cert });
    if (!result) return new BoxError(BoxError.BAD_FIELD, 'Certificate has expired.', { field: 'cert' });

    return null;
}

function reload(callback) {
    if (constants.TEST) return callback();

    shell.sudo('reload', [ RESTART_SERVICE_CMD, 'nginx' ], {}, function (error) {
        if (error) return callback(new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`));

        callback();
    });
}

function generateFallbackCertificateSync(domain) {
    assert.strictEqual(typeof domain, 'string');

    const certFilePath = path.join(os.tmpdir(), `${domain}-${crypto.randomBytes(4).readUInt32LE(0)}.cert`);
    const keyFilePath = path.join(os.tmpdir(), `${domain}-${crypto.randomBytes(4).readUInt32LE(0)}.key`);

    let opensslConf = safe.fs.readFileSync('/etc/ssl/openssl.cnf', 'utf8');
    // SAN must contain all the domains since CN check is based on implementation if SAN is found. -checkhost also checks only SAN if present!
    let opensslConfWithSan;
    let cn = domain;

    debug(`generateFallbackCertificateSync: domain=${domain} cn=${cn}`);

    opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`;
    let configFile = path.join(os.tmpdir(), 'openssl-' + crypto.randomBytes(4).readUInt32LE(0) + '.conf');
    safe.fs.writeFileSync(configFile, opensslConfWithSan, 'utf8');
    // the days field is chosen to be less than 825 days per apple requirement (https://support.apple.com/en-us/HT210176)
    let certCommand = util.format(`openssl req -x509 -newkey rsa:2048 -keyout ${keyFilePath} -out ${certFilePath} -days 800 -subj /CN=*.${cn} -extensions SAN -config ${configFile} -nodes`);
    if (!safe.child_process.execSync(certCommand)) return { error: new BoxError(BoxError.OPENSSL_ERROR, safe.error.message) };
    safe.fs.unlinkSync(configFile);

    const cert = safe.fs.readFileSync(certFilePath, 'utf8');
    if (!cert) return { error: new BoxError(BoxError.FS_ERROR, safe.error.message) };
    safe.fs.unlinkSync(certFilePath);

    const key = safe.fs.readFileSync(keyFilePath, 'utf8');
    if (!key) return { error: new BoxError(BoxError.FS_ERROR, safe.error.message) };
    safe.fs.unlinkSync(keyFilePath);

    return { cert: cert, key: key, error: null };
}

function setFallbackCertificate(domain, fallback, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert(fallback && typeof fallback === 'object');
    assert.strictEqual(typeof fallback, 'object');
    assert.strictEqual(typeof callback, 'function');

    debug(`setFallbackCertificate: setting certs for domain ${domain}`);
    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), fallback.cert)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), fallback.key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));

    // TODO: maybe the cert is being used by the mail container
    reload(callback);
}

function restoreFallbackCertificates(callback) {
    assert.strictEqual(typeof callback, 'function');

    domains.getAll(function (error, result) {
        if (error) return callback(error);

        result.forEach(function (domain) {
            if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain.domain}.host.cert`), domain.fallbackCertificate.cert)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
            if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain.domain}.host.key`), domains.fallbackCertificate.key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
        });

        callback(null);
    });
}

function getFallbackCertificatePathSync(domain) {
    assert.strictEqual(typeof domain, 'string');

    const certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`);
    const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`);

    return { certFilePath, keyFilePath };
}

function getAppCertificatePathSync(vhost) {
    assert.strictEqual(typeof vhost, 'string');

    const certFilePath = path.join(paths.NGINX_CERT_DIR, `${vhost}.user.cert`);
    const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${vhost}.user.key`);

    return { certFilePath, keyFilePath };
}

function getAcmeCertificatePathSync(vhost, domainObject) {
    assert.strictEqual(typeof vhost, 'string'); // this can contain wildcard domain (for alias domains)
    assert.strictEqual(typeof domainObject, 'object');

    let certName, certFilePath, keyFilePath, csrFilePath, acmeChallengesDir = paths.ACME_CHALLENGES_DIR;

    if (vhost !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
        certName = domains.makeWildcard(vhost).replace('*.', '_.');
        certFilePath = path.join(paths.NGINX_CERT_DIR, `${certName}.cert`);
        keyFilePath = path.join(paths.NGINX_CERT_DIR, `${certName}.key`);
        csrFilePath = path.join(paths.NGINX_CERT_DIR, `${certName}.csr`);
    } else {
        certName = vhost;
        certFilePath = path.join(paths.NGINX_CERT_DIR, `${vhost}.cert`);
        keyFilePath = path.join(paths.NGINX_CERT_DIR, `${vhost}.key`);
        csrFilePath = path.join(paths.NGINX_CERT_DIR, `${vhost}.csr`);
    }

    return { certName, certFilePath, keyFilePath, csrFilePath, acmeChallengesDir };
}

function setAppCertificate(location, domainObject, certificate, callback) {
    assert.strictEqual(typeof location, 'string');
    assert.strictEqual(typeof domainObject, 'object');
    assert.strictEqual(typeof certificate, 'object');
    assert.strictEqual(typeof callback, 'function');

    const fqdn = domains.fqdn(location, domainObject);
    const { certFilePath, keyFilePath } = getAppCertificatePathSync(fqdn);

    if (certificate.cert && certificate.key) {
        if (!safe.fs.writeFileSync(certFilePath, certificate.cert)) return callback(safe.error);
        if (!safe.fs.writeFileSync(keyFilePath, certificate.key)) return callback(safe.error);
    } else { // remove existing cert/key
        if (!safe.fs.unlinkSync(certFilePath)) debug(`Error removing cert: ${safe.error.message}`);
        if (!safe.fs.unlinkSync(keyFilePath)) debug(`Error removing key: ${safe.error.message}`);
    }

    reload(callback);
}

function getCertificatePath(fqdn, domain, callback) {
    assert.strictEqual(typeof fqdn, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

    // 1. user cert always wins
    // 2. if using fallback provider, return that cert
    // 3. look for LE certs

    domains.get(domain, function (error, domainObject) {
        if (error) return callback(error);

        const appCertPath = getAppCertificatePathSync(fqdn); // user cert always wins
        if (fs.existsSync(appCertPath.certFilePath) && fs.existsSync(appCertPath.keyFilePath)) return callback(null, appCertPath);

        if (domainObject.tlsConfig.provider === 'fallback') return callback(null, getFallbackCertificatePathSync(domain));

        const acmeCertPath = getAcmeCertificatePathSync(fqdn, domainObject);
        if (fs.existsSync(acmeCertPath.certFilePath) && fs.existsSync(acmeCertPath.keyFilePath)) return callback(null, acmeCertPath);

        return callback(null, getFallbackCertificatePathSync(domain));
    });
}

async function checkAppCertificate(vhost, domainObject) {
    assert.strictEqual(typeof vhost, 'string'); // this can contain wildcard domain (for alias domains)
    assert.strictEqual(typeof domainObject, 'object');

    const subdomain = vhost.substr(0, vhost.length - domainObject.domain.length - 1);
    const certificate = await apps.getCertificate(subdomain, domainObject.domain);
    if (!certificate) return null;

    const { certFilePath, keyFilePath } = getAppCertificatePathSync(vhost);

    if (!safe.fs.writeFileSync(certFilePath, certificate.cert)) throw new BoxError(BoxError.FS_ERROR, `Failed to write certificate: ${safe.error.message}`);
    if (!safe.fs.writeFileSync(keyFilePath, certificate.key)) throw new BoxError(BoxError.FS_ERROR, `Failed to write key: ${safe.error.message}`);

    return { certFilePath, keyFilePath };
}

async function checkAcmeCertificate(vhost, domainObject) {
    assert.strictEqual(typeof vhost, 'string'); // this can contain wildcard domain (for alias domains)
    assert.strictEqual(typeof domainObject, 'object');

    const { certName, certFilePath, keyFilePath, csrFilePath } = getAcmeCertificatePathSync(vhost, domainObject);

    const privateKey = await blobs.get(`${blobs.CERT_PREFIX}-${certName}.key`);
    const cert = await blobs.get(`${blobs.CERT_PREFIX}-${certName}.cert`);
    const csr = await blobs.get(`${blobs.CERT_PREFIX}-${certName}.csr`);

    if (!privateKey || !cert) return null;

    if (!safe.fs.writeFileSync(keyFilePath, privateKey)) throw new BoxError(BoxError.FS_ERROR, `Failed to write private key: ${safe.error.message}`);
    if (!safe.fs.writeFileSync(certFilePath, cert)) throw new BoxError(BoxError.FS_ERROR, `Failed to write certificate: ${safe.error.message}`);

    if (csr) safe.fs.writeFileSync(csrFilePath, csr);

    return { certFilePath, keyFilePath };
}

async function updateCertBlobs(vhost, domainObject) {
    assert.strictEqual(typeof vhost, 'string'); // this can contain wildcard domain (for alias domains)
    assert.strictEqual(typeof domainObject, 'object');

    const { certName, certFilePath, keyFilePath, csrFilePath } = getAcmeCertificatePathSync(vhost, domainObject);

    const privateKey = safe.fs.readFileSync(keyFilePath);
    if (!privateKey) throw new BoxError(BoxError.FS_ERROR, `Failed to read private key: ${safe.error.message}`);

    const cert = safe.fs.readFileSync(certFilePath);
    if (!cert) throw new BoxError(BoxError.FS_ERROR, `Failed to read cert: ${safe.error.message}`);

    const csr = safe.fs.readFileSync(csrFilePath);
    if (!csr) throw new BoxError(BoxError.FS_ERROR, `Failed to read csr: ${safe.error.message}`);

    await blobs.set(`${blobs.CERT_PREFIX}-${certName}.key`, privateKey);
    await blobs.set(`${blobs.CERT_PREFIX}-${certName}.cert`, cert);
    await blobs.set(`${blobs.CERT_PREFIX}-${certName}.csr`, csr);
}

function ensureCertificate(vhost, domain, auditSource, callback) {
    assert.strictEqual(typeof vhost, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    domains.get(domain, async function (error, domainObject) {
        if (error) return callback(error);

        let bundle;
        [error, bundle] = await safe(checkAppCertificate(vhost, domainObject));
        if (error) return callback(error);
        if (bundle) return callback(null, bundle, { renewed: false });

        if (domainObject.tlsConfig.provider === 'fallback') {
            debug(`ensureCertificate: ${vhost} will use fallback certs`);

            return callback(null, getFallbackCertificatePathSync(domain), { renewed: false });
        }

        getAcmeApi(domainObject, async function (error, acmeApi, apiOptions) {
            if (error) return callback(error);

            const [, currentBundle] = await safe(checkAcmeCertificate(vhost, domainObject));
            if (currentBundle) {
                debug(`ensureCertificate: ${vhost} certificate already exists at ${currentBundle.keyFilePath}`);

                if (!isExpiringSync(currentBundle.certFilePath, 24 * 30) && providerMatchesSync(domainObject, currentBundle.certFilePath, apiOptions)) return callback(null, currentBundle, { renewed: false });
                debug(`ensureCertificate: ${vhost} cert requires renewal`);
            } else {
                debug(`ensureCertificate: ${vhost} cert does not exist`);
            }

            debug('ensureCertificate: getting certificate for %s with options %j', vhost, _.omit(apiOptions, 'accountKeyPem'));

            const acmePaths = getAcmeCertificatePathSync(vhost, domainObject);
            acmeApi.getCertificate(vhost, domain, acmePaths, apiOptions, async function (error) {
                debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${acmePaths.certFilePath || 'null'}`);

                eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });

                if (error && currentBundle && !isExpiringSync(currentBundle.certFilePath, 0)) {
                    debug('ensureCertificate: continue using existing bundle since renewal failed');
                    return callback(null, currentBundle, { renewed: false });
                }

                if (!error) {
                    [error] = await safe(updateCertBlobs(vhost, domainObject));
                    if (!error) return callback(null, { certFilePath: acmePaths.certFilePath, keyFilePath: acmePaths.keyFilePath }, { renewed: true });
                }

                debug(`ensureCertificate: renewal of ${vhost} failed. using fallback certificates for ${domain}`);

                callback(null, getFallbackCertificatePathSync(domain), { renewed: false });
            });
        });
    });
}

function writeDashboardNginxConfig(bundle, configFileName, vhost, callback) {
    assert.strictEqual(typeof bundle, 'object');
    assert.strictEqual(typeof configFileName, 'string');
    assert.strictEqual(typeof vhost, 'string');
    assert.strictEqual(typeof callback, 'function');

    const data = {
        sourceDir: path.resolve(__dirname, '..'),
        vhost: vhost,
        hasIPv6: sysinfo.hasIPv6(),
        endpoint: 'dashboard',
        certFilePath: bundle.certFilePath,
        keyFilePath: bundle.keyFilePath,
        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') },
        ocsp: hasOCSPUriSync(bundle.certFilePath)
    };
    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, configFileName);

    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));

    reload(callback);
}

function writeDashboardConfig(domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

    debug(`writeDashboardConfig: writing admin config for ${domain}`);

    domains.get(domain, function (error, domainObject) {
        if (error) return callback(error);

        const dashboardFqdn = domains.fqdn(constants.DASHBOARD_LOCATION, domainObject);

        getCertificatePath(dashboardFqdn, domainObject.domain, function (error, bundle) {
            if (error) return callback(error);

            writeDashboardNginxConfig(bundle, `${dashboardFqdn}.conf`, dashboardFqdn, callback);
        });
    });
}

function writeAppNginxConfig(app, fqdn, bundle, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof fqdn, 'string');
    assert.strictEqual(typeof bundle, 'object');
    assert.strictEqual(typeof callback, 'function');

    var sourceDir = path.resolve(__dirname, '..');
    var endpoint = 'app';

    let robotsTxtQuoted = null, hideHeaders = [], cspQuoted = null;
    const reverseProxyConfig = app.reverseProxyConfig || {}; // some of our code uses fake app objects
    if (reverseProxyConfig.robotsTxt) robotsTxtQuoted = JSON.stringify(app.reverseProxyConfig.robotsTxt);
    if (reverseProxyConfig.csp) {
        cspQuoted = `"${app.reverseProxyConfig.csp}"`;
        hideHeaders = [ 'Content-Security-Policy' ];
        if (reverseProxyConfig.csp.includes('frame-ancestors ')) hideHeaders.push('X-Frame-Options');
    }

    const data = {
        sourceDir: sourceDir,
        vhost: fqdn,
        hasIPv6: sysinfo.hasIPv6(),
        ip: app.containerIp,
        port: app.manifest.httpPort,
        endpoint: endpoint,
        certFilePath: bundle.certFilePath,
        keyFilePath: bundle.keyFilePath,
        robotsTxtQuoted,
        cspQuoted,
        hideHeaders,
        proxyAuth: {
            enabled: app.sso && app.manifest.addons && app.manifest.addons.proxyAuth,
            id: app.id,
            location: nginxLocation(safe.query(app.manifest, 'addons.proxyAuth.path') || '/')
        },
        httpPaths: app.manifest.httpPaths || {},
        ocsp: hasOCSPUriSync(bundle.certFilePath)
    };
    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);

    const aliasSuffix = app.fqdn === fqdn ? '' : `-alias-${fqdn.replace('*', '_')}`;
    var nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}${aliasSuffix}.conf`);
    debug('writeAppNginxConfig: writing config for "%s" to %s with options %j', fqdn, nginxConfigFilename, data);

    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
        debug('Error creating nginx config for "%s" : %s', app.fqdn, safe.error.message);
        return callback(new BoxError(BoxError.FS_ERROR, safe.error));
    }

    reload(callback);
}

function writeAppRedirectNginxConfig(app, fqdn, bundle, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof fqdn, 'string');
    assert.strictEqual(typeof bundle, 'object');
    assert.strictEqual(typeof callback, 'function');

    const data = {
        sourceDir: path.resolve(__dirname, '..'),
        vhost: fqdn,
        redirectTo: app.fqdn,
        hasIPv6: sysinfo.hasIPv6(),
        endpoint: 'redirect',
        certFilePath: bundle.certFilePath,
        keyFilePath: bundle.keyFilePath,
        robotsTxtQuoted: null,
        cspQuoted: null,
        hideHeaders: [],
        proxyAuth: { enabled: false, id: app.id, location: nginxLocation('/') },
        ocsp: hasOCSPUriSync(bundle.certFilePath)
    };
    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);

    // if we change the filename, also change it in unconfigureApp()
    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${fqdn}.conf`);
    debug('writing config for "%s" redirecting to "%s" to %s with options %j', app.fqdn, fqdn, nginxConfigFilename, data);

    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) {
        debug('Error creating nginx redirect config for "%s" : %s', app.fqdn, safe.error.message);
        return callback(new BoxError(BoxError.FS_ERROR, safe.error));
    }

    reload(callback);
}

function writeAppConfig(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

    let appDomains = [];
    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });

    app.alternateDomains.forEach(function (alternateDomain) {
        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
    });

    app.aliasDomains.forEach(function (aliasDomain) {
        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
    });

    async.eachSeries(appDomains, function (appDomain, iteratorDone) {
        getCertificatePath(appDomain.fqdn, appDomain.domain, function (error, bundle) {
            if (error) return iteratorDone(error);

            if (appDomain.type === 'primary') {
                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
            } else if (appDomain.type === 'alternate') {
                writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
            } else if (appDomain.type === 'alias') {
                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
            }
        });
    }, callback);
}

function configureApp(app, auditSource, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof callback, 'function');

    let appDomains = [];
    appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary' });

    app.alternateDomains.forEach(function (alternateDomain) {
        appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate' });
    });

    app.aliasDomains.forEach(function (aliasDomain) {
        appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias' });
    });

    async.eachSeries(appDomains, function (appDomain, iteratorDone) {
        ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource, function (error, bundle) {
            if (error) return iteratorDone(error);

            if (appDomain.type === 'primary') {
                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
            } else if (appDomain.type === 'alternate') {
                writeAppRedirectNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
            } else if (appDomain.type === 'alias') {
                writeAppNginxConfig(app, appDomain.fqdn, bundle, iteratorDone);
            }
        });
    }, callback);
}

function unconfigureApp(app, callback) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof callback, 'function');

    // we use globbing to find all nginx configs for an app
    rimraf(path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}*.conf`), function (error) {
        if (error) debug('Error removing nginx configurations of "%s":', app.fqdn, error);

        reload(callback);
    });
}

function renewCerts(options, auditSource, progressCallback, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

    apps.getAll(function (error, allApps) {
        if (error) return callback(error);

        let appDomains = [];

        // add webadmin and mail domain
        if (settings.mailFqdn() === settings.dashboardFqdn()) {
            appDomains.push({ domain: settings.dashboardDomain(), fqdn: settings.dashboardFqdn(), type: 'webadmin+mail', nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, `${settings.dashboardFqdn()}.conf`) });
        } else {
            appDomains.push({ domain: settings.dashboardDomain(), fqdn: settings.dashboardFqdn(), type: 'webadmin', nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, `${settings.dashboardFqdn()}.conf`) });
            appDomains.push({ domain: settings.mailDomain(), fqdn: settings.mailFqdn(), type: 'mail' });
        }

        allApps.forEach(function (app) {
            if (app.runState === apps.RSTATE_STOPPED) return; // do not renew certs of stopped apps

            appDomains.push({ domain: app.domain, fqdn: app.fqdn, type: 'primary', app: app, nginxConfigFilename: path.join(paths.NGINX_APPCONFIG_DIR, app.id + '.conf') });

            app.alternateDomains.forEach(function (alternateDomain) {
                const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-redirect-${alternateDomain.fqdn}.conf`);
                appDomains.push({ domain: alternateDomain.domain, fqdn: alternateDomain.fqdn, type: 'alternate', app: app, nginxConfigFilename });
            });

            app.aliasDomains.forEach(function (aliasDomain) {
                const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `${app.id}-alias-${aliasDomain.fqdn.replace('*', '_')}.conf`);
                appDomains.push({ domain: aliasDomain.domain, fqdn: aliasDomain.fqdn, type: 'alias', app: app, nginxConfigFilename });
            });
        });

        if (options.domain) appDomains = appDomains.filter(function (appDomain) { return appDomain.domain === options.domain; });

        let progress = 1, renewed = [];

        async.eachSeries(appDomains, function (appDomain, iteratorCallback) {
            progressCallback({ percent: progress, message: `Ensuring certs of ${appDomain.fqdn}` });
            progress += Math.round(100/appDomains.length);

            ensureCertificate(appDomain.fqdn, appDomain.domain, auditSource, function (error, bundle, state) {
                if (error) return iteratorCallback(error); // this can happen if cloudron is not setup yet

                if (state.renewed) renewed.push(appDomain.fqdn);

                if (appDomain.type === 'mail') return iteratorCallback(); // mail has no nginx config to check current cert

                // hack to check if the app's cert changed or not. this doesn't handle prod/staging le change since they use same file name
                let currentNginxConfig = safe.fs.readFileSync(appDomain.nginxConfigFilename, 'utf8') || '';
                if (currentNginxConfig.includes(bundle.certFilePath)) return iteratorCallback();

                debug(`renewCerts: creating new nginx config since ${appDomain.nginxConfigFilename} does not have ${bundle.certFilePath}`);

                // reconfigure since the cert changed
                if (appDomain.type === 'webadmin' || appDomain.type === 'webadmin+mail') {
                    return writeDashboardNginxConfig(bundle, `${settings.dashboardFqdn()}.conf`, settings.dashboardFqdn(), iteratorCallback);
                } else if (appDomain.type === 'primary') {
                    return writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                } else if (appDomain.type === 'alternate') {
                    return writeAppRedirectNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                } else if (appDomain.type === 'alias') {
                    return writeAppNginxConfig(appDomain.app, appDomain.fqdn, bundle, iteratorCallback);
                }

                iteratorCallback(new BoxError(BoxError.INTERNAL_ERROR, `Unknown domain type for ${appDomain.fqdn}. This should never happen`));
            });
        }, function (error) {
            if (error) return callback(error);

            debug(`renewCerts: Renewed certs of ${JSON.stringify(renewed)}`);
            if (renewed.length === 0) return callback(null);

            async.series([
                (next) => { if (renewed.includes(settings.mailFqdn())) mail.handleCertChanged(next); else next(); }, // mail cert renewed
                reload, // reload nginx if any certs were updated but the config was not rewritten
                (next) => { // restart tls apps on cert change
                    const tlsApps = allApps.filter(app => app.manifest.addons && app.manifest.addons.tls && renewed.includes(app.fqdn));
                    async.eachSeries(tlsApps, function (app, iteratorDone) {
                        apps.restart(app, auditSource, () => iteratorDone());
                    }, next);
                }
            ], callback);
        });
    });
}

async function cleanupCerts() {
    const filenames = await fs.promises.readdir(paths.NGINX_CERT_DIR);
    const certFilenames = filenames.filter(f => f.endsWith('.cert'));

    for (const certFilename of certFilenames) {
        const certFilePath = path.join(paths.NGINX_CERT_DIR, certFilename);
        if (isExpiringSync(certFilePath, - 24 * 30 * 6)) { // expired 6 months ago
            const fqdn = certFilename.replace(/\.cert$/, '');
            debug(`cleanupCerts: deleting certs of ${fqdn}`);

            safe.fs.unlinkSync(certFilePath);
            safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${fqdn}.key`));
            safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${fqdn}.csr`));

            await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.key`);
            await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.cert`);
            await blobs.del(`${blobs.CERT_PREFIX}-${fqdn}.csr`);
        }
    }
}

function checkCerts(options, auditSource, progressCallback, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

    async.series([
        renewCerts.bind(null, options, auditSource, progressCallback),
        cleanupCerts
    ], callback);
}

function removeAppConfigs() {
    for (let appConfigFile of fs.readdirSync(paths.NGINX_APPCONFIG_DIR)) {
        if (appConfigFile !== constants.NGINX_DEFAULT_CONFIG_FILE_NAME && !appConfigFile.startsWith(constants.DASHBOARD_LOCATION)) {
            fs.unlinkSync(path.join(paths.NGINX_APPCONFIG_DIR, appConfigFile));
        }
    }
}

function writeDefaultConfig(options, callback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof callback, 'function');

    const certFilePath = path.join(paths.NGINX_CERT_DIR,  'default.cert');
    const keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key');

    if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
        debug('writeDefaultConfig: create new cert');

        const cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy
        // the days field is chosen to be less than 825 days per apple requirement (https://support.apple.com/en-us/HT210176)
        if (!safe.child_process.execSync(`openssl req -x509 -newkey rsa:2048 -keyout ${keyFilePath} -out ${certFilePath} -days 800 -subj /CN=${cn} -nodes`)) {
            debug(`writeDefaultConfig: could not generate certificate: ${safe.error.message}`);
            return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
        }
    }

    const data = {
        sourceDir: path.resolve(__dirname, '..'),
        vhost: '',
        hasIPv6: sysinfo.hasIPv6(),
        endpoint: options.activated ? 'ip' : 'setup',
        certFilePath,
        keyFilePath,
        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') },
        ocsp: false // self-signed cert
    };
    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME);

    debug(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`);

    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));

    reload(callback);
}