src/reverseproxy.js

'use strict';

exports = module.exports = {
    setUserCertificate, // per location certificate
    setFallbackCertificate, // per domain certificate

    generateFallbackCertificate,

    validateCertificate,

    getMailCertificate,
    getDirectoryServerCertificate,

    ensureCertificate,

    startRenewCerts,
    checkCerts,

    // the 'configure' functions ensure a certificate and generate nginx config
    configureApp,
    unconfigureApp,

    // these only generate nginx config
    writeDefaultConfig,
    writeDashboardConfig,
    writeAppConfigs,

    removeDashboardConfig,
    removeAppConfigs,
    restoreFallbackCertificates,

    handleCertificateProviderChanged,

    getTrustedIps,
    setTrustedIps
};

const acme2 = require('./acme2.js'),
    apps = require('./apps.js'),
    assert = require('assert'),
    blobs = require('./blobs.js'),
    BoxError = require('./boxerror.js'),
    constants = require('./constants.js'),
    crypto = require('crypto'),
    dashboard = require('./dashboard.js'),
    debug = require('debug')('box:reverseproxy'),
    dns = require('./dns.js'),
    docker = require('./docker.js'),
    domains = require('./domains.js'),
    ejs = require('ejs'),
    eventlog = require('./eventlog.js'),
    ipaddr = require('./ipaddr.js'),
    fs = require('fs'),
    Location = require('./location.js'),
    mailServer = require('./mailserver.js'),
    network = require('./network.js'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    safe = require('safetydance'),
    settings = require('./settings.js'),
    shell = require('./shell.js')('reverseproxy'),
    tasks = require('./tasks.js');

const NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/nginxconfig.ejs', { encoding: 'utf8' });
const RESTART_SERVICE_CMD = path.join(__dirname, 'scripts/restartservice.sh');

function nginxLocation(s) {
    if (!s.startsWith('!')) return s;

    const re = s.replace(/[\^$\\.*+?()[\]{}|]/g, '\\$&'); // https://github.com/es-shims/regexp.escape/blob/master/implementation.js
    return `~ ^(?!(${re.slice(1)}))`; // negative regex assertion - https://stackoverflow.com/questions/16302897/nginx-location-not-equal-to-regex
}

async function getCertificateDates(cert) {
    assert.strictEqual(typeof cert, 'string');

    const [error, result] = await safe(shell.spawn('openssl', ['x509', '-startdate', '-enddate', '-subject', '-noout'], { encoding: 'utf8', input: cert }));
    if (error) return { startDate: null, endDate: null } ; // some error

    const lines = result.trim().split('\n');
    const notBefore = lines[0].split('=')[1];
    const notBeforeDate = new Date(notBefore);

    const notAfter = lines[1].split('=')[1];
    const notAfterDate = new Date(notAfter);

    const daysLeft = (notAfterDate - new Date())/(24 * 60 * 60 * 1000);
    debug(`expiryDate: ${lines[2]} notBefore=${notBefore} notAfter=${notAfter} daysLeft=${daysLeft}`);

    return { startDate: notBeforeDate, endDate: notAfterDate };
}

async function getReverseProxyConfig() {
    const value = await settings.getJson(settings.REVERSE_PROXY_CONFIG_KEY);
    return value || { ocsp: true };
}

async function isOcspEnabled(certFilePath) {
    // on some servers, OCSP does not work. see #796
    const config = await getReverseProxyConfig();
    if (!config.ocsp) return false;

    // We used to check for the must-staple in the cert using openssl x509 -text -noout -in ${certFilePath} | grep -q status_request
    // however, we cannot set the must-staple because first request to nginx fails because of it's OCSP caching behavior
    const [error, result] = await safe(shell.spawn('openssl', ['x509', '-in', certFilePath, '-noout', '-ocsp_uri'], { encoding: 'utf8' }));
    return !error && result.length > 0; // no error and has uri
}

// checks if the certificate matches the options provided by user (like wildcard, le-staging etc)
async function providerMatches(domainObject, cert) {
    assert.strictEqual(typeof domainObject, 'object');
    assert.strictEqual(typeof cert, 'string');

    const [error, subjectAndIssuer] = await safe(shell.spawn('openssl', ['x509', '-noout', '-subject', '-issuer'], { encoding: 'utf8', input: cert }));
    if (error) return false; // something bad happenned

    const subject = subjectAndIssuer.match(/^subject=(.*)$/m)[1];
    const domain = subject.substr(subject.indexOf('=') + 1).trim(); // subject can be /CN=, CN=, CN = and other forms
    const issuer = subjectAndIssuer.match(/^issuer=(.*)$/m)[1];
    const isWildcardCert = domain.includes('*');
    const isLetsEncryptProd = issuer.includes('Let\'s Encrypt') && !issuer.includes('STAGING');

    const prod = domainObject.tlsConfig.provider.match(/.*-prod/) !== null; // matches 'le-prod' or 'letsencrypt-prod'
    const wildcard = !!domainObject.tlsConfig.wildcard;

    const issuerMismatch = (prod && !isLetsEncryptProd) || (!prod && isLetsEncryptProd);
    // bare domain is not part of wildcard SAN
    const wildcardMismatch = (domain !== domainObject.domain) && (wildcard && !isWildcardCert) || (!wildcard && isWildcardCert);

    const mismatch = issuerMismatch || wildcardMismatch;

    debug(`providerMatches: subject=${subject} domain=${domain} issuer=${issuer} `
        + `wildcard=${isWildcardCert}/${wildcard} prod=${isLetsEncryptProd}/${prod} `
        + `issuerMismatch=${issuerMismatch} wildcardMismatch=${wildcardMismatch} match=${!mismatch}`);

    return !mismatch;
}

// note: https://tools.ietf.org/html/rfc4346#section-7.4.2 (certificate_list) requires that the
// servers certificate appears first (and not the intermediate cert)
async function validateCertificate(subdomain, domain, certificate) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert(certificate && typeof certificate, 'object');

    const { cert, key } = certificate;

    // check for empty cert and key strings
    if (!cert && key) throw new BoxError(BoxError.BAD_FIELD, 'missing cert');
    if (cert && !key) throw new BoxError(BoxError.BAD_FIELD, 'missing key');

    // -checkhost checks for SAN or CN exclusively. SAN takes precedence and if present, ignores the CN.
    const fqdn = dns.fqdn(subdomain, domain);

    const [checkHostError, checkHostOutput] = await safe(shell.spawn('openssl', ['x509', '-noout', '-checkhost', fqdn], { encoding: 'utf8', input: cert }));
    if (checkHostError) throw new BoxError(BoxError.BAD_FIELD, 'Could not validate certificate');
    if (checkHostOutput.indexOf('does match certificate') === -1) throw new BoxError(BoxError.BAD_FIELD, `Certificate is not valid for this domain. Expecting ${fqdn}`);

    // check if public key in the cert and private key matches. pkey below works for RSA and ECDSA keys
    const [pubKeyError1, pubKeyFromCert] = await safe(shell.spawn('openssl', ['x509', '-noout', '-pubkey'], { encoding: 'utf8', input: cert }));
    if (pubKeyError1) throw new BoxError(BoxError.BAD_FIELD, 'Could not get public key from cert');
    const [pubKeyError2, pubKeyFromKey] = await safe(shell.spawn('openssl', ['pkey', '-pubout'], { encoding: 'utf8', input: key }));
    if (pubKeyError2) throw new BoxError(BoxError.BAD_FIELD, 'Could not get public key from private key');

    if (pubKeyFromCert !== pubKeyFromKey) throw new BoxError(BoxError.BAD_FIELD, 'Public key does not match the certificate.');

    // check expiration
    const [error] = await safe(shell.spawn('openssl', ['x509', '-checkend', '0'], { input: cert }));
    if (error) throw new BoxError(BoxError.BAD_FIELD, 'Certificate has expired');

    return null;
}

async function notifyCertChange() {
    await mailServer.checkCertificate();
    await shell.sudo([ RESTART_SERVICE_CMD, 'box' ], {}); // directory server
    const allApps = (await apps.list()).filter(app => app.runState !== apps.RSTATE_STOPPED);
    for (const app of allApps) {
        if (app.manifest.addons?.tls) await setupTlsAddon(app);
    }
}

async function reload() {
    if (constants.TEST) return;

    const [error] = await safe(shell.sudo([ RESTART_SERVICE_CMD, 'nginx' ], {}));
    if (error) throw new BoxError(BoxError.NGINX_ERROR, `Error reloading nginx: ${error.message}`);
}

// this is used in migration - 20211006200150-domains-ensure-fallbackCertificate.js
async function generateFallbackCertificate(domain) {
    assert.strictEqual(typeof domain, 'string');

    const certFilePath = path.join(os.tmpdir(), `${domain}-${crypto.randomBytes(4).readUInt32LE(0)}.cert`);
    const keyFilePath = path.join(os.tmpdir(), `${domain}-${crypto.randomBytes(4).readUInt32LE(0)}.key`);

    const opensslConf = safe.fs.readFileSync('/etc/ssl/openssl.cnf', 'utf8');
    const cn = domain;

    debug(`generateFallbackCertificate: domain=${domain} cn=${cn}`);

    // SAN must contain all the domains since CN check is based on implementation if SAN is found. -checkhost also checks only SAN if present!
    const opensslConfWithSan = `${opensslConf}\n[SAN]\nsubjectAltName=DNS:${domain},DNS:*.${cn}\n`;
    const configFile = path.join(os.tmpdir(), 'openssl-' + crypto.randomBytes(4).readUInt32LE(0) + '.conf');
    safe.fs.writeFileSync(configFile, opensslConfWithSan, 'utf8');
    // the days field is chosen to be less than 825 days per apple requirement (https://support.apple.com/en-us/HT210176)
    await shell.spawn('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-keyout', keyFilePath, '-out', certFilePath, '-days', '800', '-subj', `/CN=*.${cn}`, '-extensions', 'SAN', '-config', configFile, '-nodes'], { encoding: 'utf8 '});
    safe.fs.unlinkSync(configFile);

    const cert = safe.fs.readFileSync(certFilePath, 'utf8');
    if (!cert) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    safe.fs.unlinkSync(certFilePath);

    const key = safe.fs.readFileSync(keyFilePath, 'utf8');
    if (!key) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    safe.fs.unlinkSync(keyFilePath);

    return { cert, key };
}

async function setFallbackCertificate(domain, certificate) {
    assert.strictEqual(typeof domain, 'string');
    assert(certificate && typeof certificate === 'object');

    debug(`setFallbackCertificate: setting certs for domain ${domain}`);
    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), certificate.cert)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), certificate.key)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);

    await reload();
    await notifyCertChange(); // if domain uses fallback certs, propagate immediately
}

async function restoreFallbackCertificates() {
    const result = await domains.list();

    for (const domain of result) {
        if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain.domain}.host.cert`), domain.fallbackCertificate.cert)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
        if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain.domain}.host.key`), domain.fallbackCertificate.key)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    }
}

function getAppLocationsSync(app) {
    assert.strictEqual(typeof app, 'object');

    return [new Location(app.subdomain, app.domain, Location.TYPE_PRIMARY, app.certificate)]
        .concat(app.secondaryDomains.map(sd => new Location(sd.subdomain, sd.domain, Location.TYPE_SECONDARY, sd.certificate)))
        .concat(app.redirectDomains.map(rd => new Location(rd.subdomain, rd.domain, Location.TYPE_REDIRECT, rd.certificate)))
        .concat(app.aliasDomains.map(ad => new Location(ad.subdomain, ad.domain, Location.TYPE_ALIAS, ad.certificate)));
}

function getAcmeCertificateNameSync(fqdn, domainObject) {
    assert.strictEqual(typeof fqdn, 'string'); // this can contain wildcard domain (for alias domains)
    assert.strictEqual(typeof domainObject, 'object');

    if (fqdn !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
        return dns.makeWildcard(fqdn).replace('*.', '_.');
    } else if (fqdn.includes('*')) { // alias domain with non-wildcard cert
        return fqdn.replace('*.', '_.');
    } else {
        return fqdn;
    }
}

async function needsRenewal(cert, options) {
    assert.strictEqual(typeof cert, 'string');
    assert.strictEqual(typeof options, 'object');

    const { startDate, endDate } = await getCertificateDates(cert);
    const now = new Date();

    let isExpiring;
    if (options.forceRenewal) {
        isExpiring = (now - startDate) > (65 * 60 * 1000); // was renewed 5 minutes ago. LE backdates issue date by 1 hour for clock skew
    } else {
        isExpiring = (endDate - now) <= (30 * 24 * 60 * 60 * 1000); // expiring in a month
    }

    debug(`needsRenewal: ${isExpiring}. force: ${!!options.forceRenewal}`);
    return isExpiring;
}

async function getCertificate(location) {
    assert.strictEqual(typeof location, 'object');

    const domainObject = await domains.get(location.domain);
    if (!domainObject) throw new BoxError(BoxError.NOT_FOUND, `${location.domain} not found`);

    if (location.certificate) return location.certificate;
    if (domainObject.tlsConfig.provider === 'fallback') return domainObject.fallbackCertificate;

    const certName = getAcmeCertificateNameSync(location.fqdn, domainObject);
    const cert = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.cert`);
    const key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);
    if (!key || !cert) return domainObject.fallbackCertificate;

    return { key, cert };
}

async function getMailCertificate() {
    const mailLocation = await mailServer.getLocation();
    return await getCertificate(mailLocation);
}

async function getDirectoryServerCertificate() {
    const dashboardLocation = await dashboard.getLocation();
    return await getCertificate(dashboardLocation);
}

// write if contents mismatch (thus preserving mtime)
function writeFileSync(filePath, data) {
    assert.strictEqual(typeof filePath, 'string');
    assert.strictEqual(typeof data, 'string');

    const curData = safe.fs.readFileSync(filePath, { encoding: 'utf8' });
    if (curData === data) return false;
    if (!safe.fs.writeFileSync(filePath, data)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    return true;
}

async function setupTlsAddon(app) {
    assert.strictEqual(typeof app, 'object');

    const certificateDir = `${paths.PLATFORM_DATA_DIR}/tls/${app.id}`;
    const contents = [];
    for (const location of getAppLocationsSync(app)) {
        if (location.type === Location.TYPE_REDIRECT) continue;

        const certificate = await getCertificate(location);
        contents.push({ filename: `${location.fqdn.replace('*', '_')}.cert`, data: certificate.cert });
        contents.push({ filename: `${location.fqdn.replace('*', '_')}.key`, data: certificate.key });

        if (location.type === Location.TYPE_PRIMARY) { // backward compat
            contents.push({ filename: 'tls_cert.pem', data: certificate.cert });
            contents.push({ filename: 'tls_key.pem', data: certificate.key });
        }
    }

    let changed = 0;
    for (const content of contents) {
        if (writeFileSync(`${certificateDir}/${content.filename}`, content.data)) ++changed;
    }
    debug(`setupTlsAddon: ${changed} files changed`);

    // clean up any certs of old locations
    const filenamesInUse = new Set(contents.map(c => c.filename));
    const filenames = safe.fs.readdirSync(certificateDir) || [];
    let removed = 0;
    for (const filename of filenames) {
        if (filenamesInUse.has(filename)) continue;
        safe.fs.unlinkSync(path.join(certificateDir, filename));
        ++removed;
    }
    debug(`setupTlsAddon: ${removed} files removed`);

    if (changed || removed) await docker.restartContainer(app.id);
}

// writes latest certificate to disk and returns the path
async function writeCertificate(location) {
    assert.strictEqual(typeof location, 'object');

    const { domain, fqdn } = location;
    const domainObject = await domains.get(domain);
    if (!domainObject) throw new BoxError(BoxError.NOT_FOUND, `${domain} not found`);

    if (location.certificate) {
        const certFilePath = path.join(paths.NGINX_CERT_DIR, `${fqdn}.user.cert`);
        const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${fqdn}.user.key`);

        writeFileSync(certFilePath, location.certificate.cert);
        writeFileSync(keyFilePath, location.certificate.key);

        return { certFilePath, keyFilePath };
    }

    if (domainObject.tlsConfig.provider === 'fallback') {
        const certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`);
        const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`);

        debug(`writeCertificate: ${fqdn} will use fallback certs`);
        writeFileSync(certFilePath, domainObject.fallbackCertificate.cert);
        writeFileSync(keyFilePath, domainObject.fallbackCertificate.key);

        return { certFilePath, keyFilePath };
    }

    const certName = getAcmeCertificateNameSync(fqdn, domainObject);
    let cert = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.cert`);
    let key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);

    if (!key || !cert) { // use fallback certs if we didn't manage to get acme certs
        debug(`writeCertificate: ${fqdn} will use fallback certs because acme is missing`);
        cert = domainObject.fallbackCertificate.cert;
        key = domainObject.fallbackCertificate.key;
    }
    const certFilePath = path.join(paths.NGINX_CERT_DIR, `${certName}.cert`);
    const keyFilePath = path.join(paths.NGINX_CERT_DIR, `${certName}.key`);

    writeFileSync(certFilePath, cert);
    writeFileSync(keyFilePath, key);

    return { certFilePath, keyFilePath };
}

async function ensureCertificate(location, options, auditSource) {
    assert.strictEqual(typeof location, 'object');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const domainObject = await domains.get(location.domain);
    if (!domainObject) throw new BoxError(BoxError.NOT_FOUND, `${location.domain} not found`);

    const fqdn = dns.fqdn(location.subdomain, location.domain);

    if (location.certificate) { // user certificate
        debug(`ensureCertificate: ${fqdn} will use user certs`);
        return;
    }

    if (domainObject.tlsConfig.provider === 'fallback') {
        debug(`ensureCertificate: ${fqdn} will use fallback certs`);
        return;
    }

    const certName = getAcmeCertificateNameSync(fqdn, domainObject);
    const key = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.key`);
    const cert = await blobs.getString(`${blobs.CERT_PREFIX}-${certName}.cert`);

    if (key && cert) {
        const sameProvider = await providerMatches(domainObject, cert);
        const outdated = await needsRenewal(cert, options);
        if (sameProvider && !outdated) {
            debug(`ensureCertificate: ${fqdn} acme cert exists and is up to date`);
            return;
        }
        debug(`ensureCertificate: ${fqdn} acme cert exists but provider mismatch or needs renewal`);
    }

    debug(`ensureCertificate: ${fqdn} needs acme cert`);
    const [error] = await safe(acme2.getCertificate(fqdn, domainObject));
    debug(`ensureCertificate: error: ${error ? error.message : 'null'}`);

    await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: fqdn, errorMessage: error?.message || '' }));
}

async function writeDashboardNginxConfig(vhost, certificatePath) {
    assert.strictEqual(typeof vhost, 'string');
    assert.strictEqual(typeof certificatePath, 'object');

    const data = {
        sourceDir: path.resolve(__dirname, '..'),
        vhost,
        hasIPv6: network.hasIPv6(),
        endpoint: 'dashboard',
        certFilePath: certificatePath.certFilePath,
        keyFilePath: certificatePath.keyFilePath,
        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') },
        ocsp: await isOcspEnabled(certificatePath.certFilePath),
        hstsPreload: false
    };
    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `dashboard/${vhost}.conf`);

    writeFileSync(nginxConfigFilename, nginxConf);
}

// also syncs the certs to disk
async function writeDashboardConfig(subdomain, domain) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');

    debug(`writeDashboardConfig: writing dashboard config for ${domain}`);

    const dashboardFqdn = dns.fqdn(subdomain, domain);
    const location = { domain, fqdn: dashboardFqdn, certificate: null };
    const certificatePath = await writeCertificate(location);
    await writeDashboardNginxConfig(dashboardFqdn, certificatePath);
    await reload();
}

async function removeDashboardConfig(subdomain, domain) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');

    debug(`removeDashboardConfig: removing dashboard config of ${domain}`);

    const vhost = dns.fqdn(subdomain, domain);
    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, `dashboard/${vhost}.conf`);

    if (!safe.fs.unlinkSync(nginxConfigFilename)) throw new BoxError(BoxError.FS_ERROR, safe.error.message);

    await reload();
}

async function writeAppLocationNginxConfig(app, location, certificatePath) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof location, 'object');
    assert.strictEqual(typeof certificatePath, 'object');

    const { type, fqdn} = location;

    const data = {
        sourceDir: path.resolve(__dirname, '..'),
        vhost: fqdn,
        hasIPv6: network.hasIPv6(),
        ip: null,
        port: null,
        endpoint: null,
        redirectTo: null,
        certFilePath: certificatePath.certFilePath,
        keyFilePath: certificatePath.keyFilePath,
        robotsTxtQuoted: null,
        cspQuoted: null,
        hideHeaders: [],
        proxyAuth: { enabled: false },
        upstreamUri: '', // only for endpoint === external
        ocsp: await isOcspEnabled(certificatePath.certFilePath),
        hstsPreload: !!app.reverseProxyConfig?.hstsPreload
    };

    if (type === Location.TYPE_PRIMARY || type === Location.TYPE_ALIAS || type === Location.TYPE_SECONDARY) {
        data.endpoint = 'app';

        if (app.manifest.id === constants.PROXY_APP_APPSTORE_ID) {
            data.endpoint = 'external';
            data.upstreamUri = app.upstreamUri;
        }

        // maybe these should become per domain at some point
        const reverseProxyConfig = app.reverseProxyConfig || {}; // some of our code uses fake app objects
        if (reverseProxyConfig.robotsTxt) data.robotsTxtQuoted = JSON.stringify(app.reverseProxyConfig.robotsTxt);
        if (reverseProxyConfig.csp) {
            data.cspQuoted = `"${app.reverseProxyConfig.csp}"`;
            data.hideHeaders = [ 'Content-Security-Policy' ];
            if (reverseProxyConfig.csp.includes('frame-ancestors ')) data.hideHeaders.push('X-Frame-Options');
        }
        if (type === Location.TYPE_PRIMARY || type == Location.TYPE_ALIAS) {
            data.proxyAuth = {
                enabled: app.sso && app.manifest.addons && app.manifest.addons.proxyAuth,
                id: app.id,
                location: nginxLocation(safe.query(app.manifest, 'addons.proxyAuth.path') || '/')
            };
            data.ip = app.containerIp;
            data.port = app.manifest.httpPort;

            if (data.proxyAuth.enabled) {
                data.proxyAuth.oidcClientId = app.id;
                data.proxyAuth.oidcEndpoint = (await dashboard.getLocation()).fqdn;
            }
        } else if (type === Location.TYPE_SECONDARY) {
            data.ip = app.containerIp;
            const secondaryDomain = app.secondaryDomains.find(sd => sd.fqdn === fqdn);
            data.port = app.manifest.httpPorts[secondaryDomain.environmentVariable].containerPort;
        }
    } else if (type === Location.TYPE_REDIRECT) {
        data.proxyAuth = { enabled: false, id: app.id, location: nginxLocation('/') };
        data.endpoint = 'redirect';
        data.redirectTo = app.fqdn;
    }

    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
    const filename = path.join(paths.NGINX_APPCONFIG_DIR, app.id, `${fqdn.replace('*', '_')}.conf`);
    debug(`writeAppLocationNginxConfig: writing config for "${fqdn}" to ${filename} with options ${JSON.stringify(data)}`);
    writeFileSync(filename, nginxConf);
}

async function writeAppConfigs(app) {
    assert.strictEqual(typeof app, 'object');

    const locations = getAppLocationsSync(app);

    if (!safe.fs.mkdirSync(path.join(paths.NGINX_APPCONFIG_DIR, app.id), { recursive: true })) throw new BoxError(BoxError.FS_ERROR, `Could not create nginx config directory: ${safe.error.message}`);

    for (const location of locations) {
        const certificatePath = await writeCertificate(location);
        await writeAppLocationNginxConfig(app, location, certificatePath);
    }

    await reload();
}

async function setUserCertificate(app, location) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof location, 'object');

    const certificatePath = await writeCertificate(location);
    await writeAppLocationNginxConfig(app, location, certificatePath);
    await reload();
}

async function configureApp(app, auditSource) {
    assert.strictEqual(typeof app, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const locations = getAppLocationsSync(app);

    for (const location of locations) {
        await ensureCertificate(location, {}, auditSource);
    }

    await writeAppConfigs(app);

    if (app.manifest.addons?.tls) await setupTlsAddon(app);
}

async function unconfigureApp(app) {
    assert.strictEqual(typeof app, 'object');

    if (!safe.fs.rmSync(path.join(paths.NGINX_APPCONFIG_DIR, app.id), { recursive: true, force: true })) throw new BoxError(BoxError.FS_ERROR, `Could not remove nginx config directory: ${safe.error.message}`);
    await reload();
}

async function cleanupCerts(locations, auditSource, progressCallback) {
    assert(Array.isArray(locations));
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    progressCallback({ message: 'Checking expired certs for removal' });

    const domainObjectMap = await domains.getDomainObjectMap();
    const certNamesInUse = new Set();
    for (const location of locations) {
        certNamesInUse.add(await getAcmeCertificateNameSync(location.fqdn, domainObjectMap[location.domain]));
    }

    const now = new Date();
    const certIds = await blobs.listCertIds();
    const removedCertNames = [];
    for (const certId of certIds) {
        const certName = certId.match(new RegExp(`${blobs.CERT_PREFIX}-(.*).cert`))[1];
        if (certNamesInUse.has(certName)) continue;

        const cert = await blobs.getString(certId);
        const { endDate } = await getCertificateDates(cert);
        if (!endDate) continue; // some error

        if (now - endDate >= (60 * 60 * 24 * 30 * 6 * 1000)) { // expired 6 months ago and not in use
            progressCallback({ message: `deleting certs of ${certName}` });

            // it is safe to delete the certs of stopped apps because their nginx configs are removed
            safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${certName}.cert`));
            safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${certName}.key`));
            safe.fs.unlinkSync(path.join(paths.NGINX_CERT_DIR, `${certName}.csr`));

            await blobs.del(`${blobs.CERT_PREFIX}-${certName}.key`);
            await blobs.del(`${blobs.CERT_PREFIX}-${certName}.cert`);
            await blobs.del(`${blobs.CERT_PREFIX}-${certName}.csr`);

            removedCertNames.push(certName);
        }
    }

    if (removedCertNames.length) await safe(eventlog.add(eventlog.ACTION_CERTIFICATE_CLEANUP, auditSource, { domains: removedCertNames }));

    debug('cleanupCerts: done');
}

async function checkCerts(options, auditSource, progressCallback) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    let locations = [];
    const dashboardLocation = await dashboard.getLocation();
    locations.push(dashboardLocation);

    const mailLocation = await mailServer.getLocation();
    if (dashboardLocation.fqdn !== mailLocation.fqdn) locations.push(mailLocation);

    const allApps = (await apps.list()).filter(app => app.runState !== apps.RSTATE_STOPPED);
    for (const app of allApps) {
        locations = locations.concat(getAppLocationsSync(app));
    }

    let percent = 1;
    for (const location of locations) {
        percent += Math.round(100/locations.length);
        progressCallback({ percent, message: `Ensuring certs of ${location.fqdn}` });
        await ensureCertificate(location, options, auditSource);
    }

    if (options.rebuild || fs.existsSync(paths.REVERSE_PROXY_REBUILD_FILE)) {
        progressCallback( { message: 'Rebuilding app configs' });
        for (const app of allApps) {
            await writeAppConfigs(app);
        }
        await writeDashboardConfig(dashboardLocation.subdomain, dashboardLocation.domain);
        await notifyCertChange(); // this allows user to "rebuild" using UI just in case we crashed and went out of sync
        safe.fs.unlinkSync(paths.REVERSE_PROXY_REBUILD_FILE);
    } else {
        // sync all locations and not just the ones that changed. this helps with 0 length certs when disk is full and also
        // if renewal task crashed midway.
        for (const location of locations) {
            await writeCertificate(location);
        }
        await reload();
        await notifyCertChange(); // propagate any cert changes to services
    }

    await cleanupCerts(locations, auditSource, progressCallback);
}

async function startRenewCerts(options, auditSource) {
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const taskId = await tasks.add(tasks.TASK_CHECK_CERTS, [ options, auditSource ]);
    safe(tasks.startTask(taskId, {}), { debug }); // background
    return taskId;
}

function removeAppConfigs() {
    debug('removeAppConfigs: removing app nginx configs');

    // remove all configs which are not the default or current dashboard
    for (const entry of fs.readdirSync(paths.NGINX_APPCONFIG_DIR, { withFileTypes: true })) {
        if (entry.isDirectory() && entry.name === 'dashboard') continue;
        if (entry.isFile() && entry.name === constants.NGINX_DEFAULT_CONFIG_FILE_NAME) continue;

        const fullPath = path.join(paths.NGINX_APPCONFIG_DIR, entry.name);
        if (entry.isDirectory()) {
            fs.rmSync(fullPath, { recursive: true, force: true });
        } else if (entry.isFile()) {
            fs.unlinkSync(fullPath);
        }
    }
}

async function writeDefaultConfig(options) {
    assert.strictEqual(typeof options, 'object');

    const certFilePath = path.join(paths.NGINX_CERT_DIR,  'default.cert');
    const keyFilePath = path.join(paths.NGINX_CERT_DIR, 'default.key');

    if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
        debug('writeDefaultConfig: create new cert');

        const cn = 'cloudron-' + (new Date()).toISOString(); // randomize date a bit to keep firefox happy
        // the days field is chosen to be less than 825 days per apple requirement (https://support.apple.com/en-us/HT210176)
        await shell.spawn('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-keyout', keyFilePath, '-out', certFilePath, '-days', '800', '-subj', `/CN=${cn}`, '-nodes'], { encoding: 'utf8' });
    }

    const data = {
        sourceDir: path.resolve(__dirname, '..'),
        vhost: '',
        hasIPv6: network.hasIPv6(),
        endpoint: options.activated ? 'ip' : 'setup',
        certFilePath,
        keyFilePath,
        robotsTxtQuoted: JSON.stringify('User-agent: *\nDisallow: /\n'),
        proxyAuth: { enabled: false, id: null, location: nginxLocation('/') },
        ocsp: false, // self-signed cert
        hstsPreload: false
    };
    const nginxConf = ejs.render(NGINX_APPCONFIG_EJS, data);
    const nginxConfigFilename = path.join(paths.NGINX_APPCONFIG_DIR, constants.NGINX_DEFAULT_CONFIG_FILE_NAME);

    debug(`writeDefaultConfig: writing configs for endpoint "${data.endpoint}"`);

    if (!safe.fs.writeFileSync(nginxConfigFilename, nginxConf)) throw new BoxError(BoxError.FS_ERROR, safe.error);

    await reload();
}

async function handleCertificateProviderChanged(domain) {
    assert.strictEqual(typeof domain, 'string');

    safe.fs.appendFileSync(paths.REVERSE_PROXY_REBUILD_FILE, `${domain}\n`, 'utf8');
}

async function getTrustedIps() {
    const value = await settings.get(settings.TRUSTED_IPS_KEY);
    return value || '';
}

async function setTrustedIps(trustedIps) {
    assert.strictEqual(typeof trustedIps, 'string');

    let trustedIpsConfig = 'real_ip_header X-Forwarded-For;\nreal_ip_recursive on;\n';

    for (const line of trustedIps.split('\n')) {
        if (!line || line.startsWith('#')) continue;
        const rangeOrIP = line.trim();
        if (!ipaddr.isValid(rangeOrIP) && !ipaddr.isValidCIDR(rangeOrIP)) throw new BoxError(BoxError.BAD_FIELD, `${rangeOrIP} is not a valid IP or range`);
        trustedIpsConfig += `set_real_ip_from ${rangeOrIP};\n`;
    }

    await settings.set(settings.TRUSTED_IPS_KEY, trustedIps);
    if (!safe.fs.writeFileSync(paths.NGINX_TRUSTED_IPS_FILE, trustedIpsConfig, 'utf8')) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    await reload();
}