split mail and mailserver

mail = all the per-domain code mailserver = all the mail server level code
2023-08-04 20:54:16 +05:30
parent fb9d8c23e1
commit 946e5caacb
8 changed files with 349 additions and 327 deletions
--- a/src/mailserver.js
+++ b/src/mailserver.js
@@ -0,0 +1,322 @@
+'use strict';
+
+exports = module.exports = {
+    restart,
+    start,
+
+    generateDkimKey,
+
+    onDomainAdded,
+    onDomainRemoved,
+
+    checkCertificate,
+
+    getMailAuth,
+
+    getLocation,
+    setLocation, // triggers the change task
+    changeLocation, // does the actual changing
+
+    DEFAULT_MEMORY_LIMIT: 512 * 1024 * 1024,
+};
+
+const assert = require('assert'),
+    BoxError = require('./boxerror.js'),
+    cloudron = require('./cloudron.js'),
+    constants = require('./constants.js'),
+    crypto = require('crypto'),
+    debug = require('debug')('box:mailserver'),
+    dns = require('./dns.js'),
+    docker = require('./docker.js'),
+    domains = require('./domains.js'),
+    eventlog = require('./eventlog.js'),
+    hat = require('./hat.js'),
+    infra = require('./infra_version.js'),
+    mail = require('./mail.js'),
+    os = require('os'),
+    path = require('path'),
+    paths = require('./paths.js'),
+    reverseProxy = require('./reverseproxy.js'),
+    safe = require('safetydance'),
+    services = require('./services.js'),
+    settings = require('./settings.js'),
+    shell = require('./shell.js'),
+    system = require('./system.js'),
+    tasks = require('./tasks.js'),
+    users = require('./users.js');
+
+async function generateDkimKey() {
+    const publicKeyFilePath = path.join(os.tmpdir(), `dkim-${crypto.randomBytes(4).readUInt32LE(0)}.public`);
+    const privateKeyFilePath = path.join(os.tmpdir(), `dkim-${crypto.randomBytes(4).readUInt32LE(0)}.private`);
+
+    // https://www.unlocktheinbox.com/dkim-key-length-statistics/ and https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-authentication-dkim-easy.html for key size
+    if (!safe.child_process.execSync(`openssl genrsa -out ${privateKeyFilePath} 1024`)) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+    if (!safe.child_process.execSync(`openssl rsa -in ${privateKeyFilePath} -out ${publicKeyFilePath} -pubout -outform PEM`)) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);
+
+    const publicKey = safe.fs.readFileSync(publicKeyFilePath, 'utf8');
+    if (!publicKey) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
+    safe.fs.unlinkSync(publicKeyFilePath);
+
+    const privateKey = safe.fs.readFileSync(privateKeyFilePath, 'utf8');
+    if (!privateKey) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
+    safe.fs.unlinkSync(privateKeyFilePath);
+
+    return { publicKey, privateKey };
+}
+
+async function createMailConfig(mailFqdn) {
+    assert.strictEqual(typeof mailFqdn, 'string');
+
+    debug(`createMailConfig: generating mail config with ${mailFqdn}`);
+
+    const mailDomains = await mail.listDomains();
+
+    const mailOutDomains = mailDomains.filter(d => d.relay.provider !== 'noop').map(d => d.domain).join(',');
+    const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');
+
+    // mail_domain is used for SRS
+    if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/mail.ini`,
+        `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\n\n`, 'utf8')) {
+        throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
+    }
+
+    // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
+    if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/smtp_forward.ini`, 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
+        throw new BoxError(BoxError.FS_ERROR, `Could not create smtp forward file: ${safe.error.message}`);
+    }
+
+    // create sections for per-domain configuration
+    for (const domain of mailDomains) {
+        const catchAll = domain.catchAll.join(',');
+        const mailFromValidation = domain.mailFromValidation;
+
+        if (!safe.fs.appendFileSync(`${paths.MAIL_CONFIG_DIR}/mail.ini`,
+            `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
+            throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
+        }
+
+        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/banner/${domain.domain}.text`, domain.banner.text || '')) throw new BoxError(BoxError.FS_ERROR, `Could not create text banner file: ${safe.error.message}`);
+        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/banner/${domain.domain}.html`, domain.banner.html || '')) throw new BoxError(BoxError.FS_ERROR, `Could not create html banner file: ${safe.error.message}`);
+
+        safe.fs.mkdirSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}`, { recursive: true });
+        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/public`, domain.dkimKey.publicKey)) throw new BoxError(BoxError.FS_ERROR, `Could not create public key file: ${safe.error.message}`);
+        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/private`, domain.dkimKey.privateKey)) throw new BoxError(BoxError.FS_ERROR, `Could not create private key file: ${safe.error.message}`);
+        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/selector`, domain.dkimSelector)) throw new BoxError(BoxError.FS_ERROR, `Could not create selector file: ${safe.error.message}`);
+
+        // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
+        if (!safe.fs.chmodSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/private`, 0o644)) throw new BoxError(BoxError.FS_ERROR, safe.error);
+
+        const relay = domain.relay;
+
+        const enableRelay = relay.provider !== 'cloudron-smtp' && relay.provider !== 'noop',
+            host = relay.host || '',
+            port = relay.port || 25,
+            // office365 removed plain auth (https://support.microsoft.com/en-us/office/outlook-com-no-longer-supports-auth-plain-authentication-07f7d5e9-1697-465f-84d2-4513d4ff0145)
+            authType = relay.username ? (relay.provider === 'office365-legacy-smtp' ? 'login' : 'plain') : '',
+            username = relay.username || '',
+            password = relay.password || '',
+            forceFromAddress = relay.forceFromAddress ? 'true' : 'false';
+
+        if (!enableRelay) continue;
+
+        const relayData = `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\nforce_from_address=${forceFromAddress}\n\n`;
+
+        if (!safe.fs.appendFileSync(paths.MAIL_CONFIG_DIR + '/smtp_forward.ini', relayData, 'utf8')) {
+            throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
+        }
+    }
+
+    return mailInDomains.length !== 0 /* allowInbound */;
+}
+
+async function configureMail(mailFqdn, mailDomain, serviceConfig) {
+    assert.strictEqual(typeof mailFqdn, 'string');
+    assert.strictEqual(typeof mailDomain, 'string');
+    assert.strictEqual(typeof serviceConfig, 'object');
+
+    // mail (note: 2587 is hardcoded in mail container and app use this port)
+    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
+    // MAIL_DOMAIN is the domain for which this server is relaying mails
+    // mail container uses /app/data for backed up data and /run for restart-able data
+
+    const tag = infra.images.mail.tag;
+    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
+    const memory = await system.getMemoryAllocation(memoryLimit);
+    const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);
+
+    const certificate = await reverseProxy.getMailCertificate();
+
+    const dhparamsFilePath = `${paths.MAIL_CONFIG_DIR}/dhparams.pem`;
+    const mailCertFilePath = `${paths.MAIL_CONFIG_DIR}/tls_cert.pem`;
+    const mailKeyFilePath =  `${paths.MAIL_CONFIG_DIR}/tls_key.pem`;
+
+    if (!safe.child_process.execSync(`cp ${paths.DHPARAMS_FILE} ${dhparamsFilePath}`)) throw new BoxError(BoxError.FS_ERROR, `Could not copy dhparams: ${safe.error.message}`);
+    if (!safe.fs.writeFileSync(mailCertFilePath, certificate.cert)) throw new BoxError(BoxError.FS_ERROR, `Could not create cert file: ${safe.error.message}`);
+    if (!safe.fs.writeFileSync(mailKeyFilePath, certificate.key)) throw new BoxError(BoxError.FS_ERROR, `Could not create key file: ${safe.error.message}`);
+
+    // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
+    if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`);
+
+    await shell.promises.exec('stopMail', 'docker stop mail || true');
+    await shell.promises.exec('removeMail', 'docker rm -f mail || true');
+
+    const allowInbound = await createMailConfig(mailFqdn);
+
+    const ports = allowInbound ? '-p 587:2587 -p 993:9993 -p 4190:4190 -p 25:2587 -p 465:2465 -p 995:9995' : '';
+    const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
+    const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : '';
+    const logLevel = serviceConfig.recoveryMode ? 'data' : 'info';
+
+    const runCmd = `docker run --restart=always -d --name="mail" \
+                --net cloudron \
+                --net-alias mail \
+                --log-driver syslog \
+                --log-opt syslog-address=udp://127.0.0.1:2514 \
+                --log-opt syslog-format=rfc5424 \
+                --log-opt tag=mail \
+                -m ${memory} \
+                --memory-swap ${memoryLimit} \
+                --dns 172.18.0.1 \
+                --dns-search=. \
+                -e CLOUDRON_MAIL_TOKEN="${cloudronToken}" \
+                -e CLOUDRON_RELAY_TOKEN="${relayToken}" \
+                -e LOGLEVEL=${logLevel} \
+                -v "${paths.MAIL_DATA_DIR}:/app/data" \
+                -v "${paths.MAIL_CONFIG_DIR}:/etc/mail:ro" \
+                ${ports} \
+                --label isCloudronManaged=true \
+                ${readOnly} -v /run -v /tmp ${tag} ${cmd}`;
+
+    await shell.promises.exec('startMail', runCmd);
+}
+
+async function restart() {
+    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return;
+
+    const mailConfig = await services.getServiceConfig('mail');
+    debug(`restart: restarting mail container with mailFqdn:${settings.mailFqdn()} mailDomain:${settings.mailDomain()}`);
+    await configureMail(settings.mailFqdn(), settings.mailDomain(), mailConfig);
+}
+
+async function start(existingInfra) {
+    assert.strictEqual(typeof existingInfra, 'object');
+
+    debug('startMail: starting');
+    await restart();
+}
+
+async function restartIfActivated() {
+    const activated = await users.isActivated();
+
+    if (!activated) {
+        debug('restartMailIfActivated: skipping restart of mail container since Cloudron is not activated yet');
+        return; // not provisioned yet, do not restart container after dns setup
+    }
+
+    debug('restartMailIfActivated: restarting on activated');
+    await restart();
+}
+
+async function onDomainAdded(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    if (!settings.mailFqdn()) return; // mail domain is not set yet (when provisioning)
+
+    debug(`onDomainAdded: configuring mail for added domain ${domain}`);
+    await mail.upsertDnsRecords(domain, settings.mailFqdn());
+    await restartIfActivated();
+}
+
+async function onDomainRemoved(domain) {
+    assert.strictEqual(typeof domain, 'string');
+
+    debug(`onDomainRemoved: configuring mail for removed domain ${domain}`);
+    await restart();
+}
+
+async function checkCertificate() {
+    const certificate = await reverseProxy.getMailCertificate();
+    const cert = safe.fs.readFileSync(`${paths.MAIL_CONFIG_DIR}/tls_cert.pem`, { encoding: 'utf8' });
+    if (cert === certificate.cert) {
+        debug('checkCertificate: certificate has not changed');
+        return;
+    }
+    debug('checkCertificate: certificate has changed');
+
+    await restartIfActivated();
+}
+
+async function getLocation() {
+    const domain = settings.mailDomain(), fqdn = settings.mailFqdn();
+    const subdomain = fqdn.substr(0, fqdn.length - domain.length - 1);
+
+    return { domain, subdomain };
+}
+
+async function changeLocation(auditSource, progressCallback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+
+    const fqdn = settings.mailFqdn(), domain = settings.mailDomain();
+    const subdomain = fqdn.substr(0, fqdn.length - domain.length - 1);
+
+    let progress = 20;
+    progressCallback({ percent: progress, message: `Setting up DNS of certs of mail server ${fqdn}` });
+
+    await cloudron.setupDnsAndCert(subdomain, domain, auditSource, (progress) => progressCallback({ message: progress.message })); // remove the percent
+    const allDomains = await domains.list();
+
+    for (let idx = 0; idx < allDomains.length; idx++) {
+        const domainObject = allDomains[idx];
+        progressCallback({ percent: progress, message: `Updating DNS of ${domainObject.domain}` });
+        progress += Math.round(70/allDomains.length);
+
+        const [error] = await safe(mail.upsertDnsRecords(domainObject.domain, fqdn)); // ignore any errors. we anyway report dns errors in status tab
+        progressCallback({ percent: progress, message: `Updated DNS of ${domainObject.domain}: ${error ? error.message : 'success'}` });
+    }
+
+    progressCallback({ percent: 90, message: 'Restarting mail server' });
+
+    await restartIfActivated();
+}
+
+async function setLocation(subdomain, domain, auditSource) {
+    assert.strictEqual(typeof subdomain, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof auditSource, 'object');
+
+    const domainObjectMap = await domains.getDomainObjectMap();
+    if (!(domain in domainObjectMap)) throw new BoxError(BoxError.BAD_FIELD, `No such domain '${domain}'`);
+    const error = dns.validateHostname(subdomain, domain);
+    if (error) throw new BoxError(BoxError.BAD_FIELD, `Bad mail location: ${error.message}`);
+
+    const fqdn = dns.fqdn(subdomain, domain);
+
+    await settings.setMailLocation(domain, fqdn);
+
+    const taskId = await tasks.add(tasks.TASK_CHANGE_MAIL_LOCATION, [ auditSource ]);
+    tasks.startTask(taskId, {});
+    await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId });
+
+    return taskId;
+}
+
+async function getMailAuth() {
+    const data = await docker.inspect('mail');
+    const ip = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
+    if (!ip) throw new BoxError(BoxError.MAIL_ERROR, 'Error querying mail server IP');
+
+    // extract the relay token for auth
+    const env = safe.query(data, 'Config.Env', null);
+    if (!env) throw new BoxError(BoxError.MAIL_ERROR, 'Error getting mail env');
+    const tmp = env.find(function (e) { return e.indexOf('CLOUDRON_RELAY_TOKEN') === 0; });
+    if (!tmp) throw new BoxError(BoxError.MAIL_ERROR, 'Error getting CLOUDRON_RELAY_TOKEN env var');
+    const relayToken = tmp.slice('CLOUDRON_RELAY_TOKEN'.length + 1); // +1 for the = sign
+    if (!relayToken) throw new BoxError(BoxError.MAIL_ERROR, 'Error parsing CLOUDRON_RELAY_TOKEN');
+
+    return {
+        ip,
+        port: constants.INTERNAL_SMTP_PORT,
+        relayToken
+    };
+}