cloudron-box/src/mail.js

import assert from 'node:assert';
import BoxError from './boxerror.js';
import constants from './constants.js';
import database from './database.js';
import logger from './logger.js';
import dig from './dig.js';
import dns from './dns.js';
import eventlog from './eventlog.js';
import groups from './groups.js';
import mailer from './mailer.js';
import mailServer from './mailserver.js';
import net from 'node:net';
import network from './network.js';
import nodemailer from 'nodemailer';
import notifications from './notifications.js';
import path from 'node:path';
import platform from './platform.js';
import safe from '@cloudron/safetydance';
import services from './services.js';
import shellModule from './shell.js';
import superagent from '@cloudron/superagent';
import validator from './validator.js';
import _ from './underscore.js';

const { log } = logger('mail');
const shell = shellModule('mail');

const OWNERTYPE_USER = 'user';
const OWNERTYPE_GROUP = 'group';
const OWNERTYPE_APP = 'app';
const TYPE_MAILBOX = 'mailbox';
const TYPE_LIST = 'list';
const TYPE_ALIAS = 'alias';


const DNS_OPTIONS = { timeout: 20000, tries: 4 };
const REMOVE_MAILBOX_CMD = path.join(import.meta.dirname, 'scripts/rmmailbox.sh');

// if you add a field here, listMailboxes* has to be updated
const MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active', 'enablePop3', 'storageQuota', 'messagesQuota' ].join(',');
const MAILDB_FIELDS = [ 'domain', 'enabled', 'mailFromValidation', 'catchAllJson', 'relayJson', 'dkimKeyJson', 'dkimSelector', 'bannerJson' ].join(',');

function postProcessMailbox(data) {
    data.members = safe.JSON.parse(data.membersJson) || [ ];
    delete data.membersJson;

    data.membersOnly = !!data.membersOnly;
    data.active = !!data.active;
    data.enablePop3 = !!data.enablePop3;

    return data;
}

function postProcessAliases(data) {
    const aliasNames = JSON.parse(data.aliasNames), aliasDomains = JSON.parse(data.aliasDomains);
    delete data.aliasNames;
    delete data.aliasDomains;
    data.aliases = [];
    for (let i = 0; i < aliasNames.length; i++) { // NOTE: aliasNames is [ null ] when no aliases
        if (aliasNames[i]) data.aliases[i] = { name: aliasNames[i], domain: aliasDomains[i] };
    }

    return data;
}

function postProcessDomain(data) {
    data.enabled = !!data.enabled; // int to boolean
    data.mailFromValidation = !!data.mailFromValidation; // int to boolean

    data.catchAll = safe.JSON.parse(data.catchAllJson) || [ ];
    delete data.catchAllJson;

    data.relay = safe.JSON.parse(data.relayJson) || { provider: 'cloudron-smtp' };
    delete data.relayJson;

    data.banner = safe.JSON.parse(data.bannerJson) || { text: null, html: null };
    delete data.bannerJson;

    data.dkimKey = safe.JSON.parse(data.dkimKeyJson) || null;
    delete data.dkimKeyJson;

    return data;
}

function validateName(name) {
    assert.strictEqual(typeof name, 'string');

    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox name must be atleast 1 char');
    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');

    // also need to consider valid LDAP characters here (e.g '+' is reserved). keep hyphen at the end so it doesn't become a range.
    if (/[^a-zA-Z0-9._-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals, dot, hyphen or underscore');

    return null;
}

function validateAlias(name) {
    assert.strictEqual(typeof name, 'string');

    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox name must be atleast 1 char');
    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');

    // also need to consider valid LDAP characters here (e.g '+' is reserved). keep hyphen at the end so it doesn't become a range.
    if (/[^a-zA-Z0-9._*-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals, dot, hyphen, asterisk or underscore');

    return null;
}

function validateDisplayName(name) {
    assert.strictEqual(typeof name, 'string');

    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name must be atleast 1 char');
    if (name.length >= 100) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name too long');
    // technically only ":" is disallowed it seems (https://www.rfc-editor.org/rfc/rfc5322#section-2.2)
    // in https://www.rfc-editor.org/rfc/rfc2822.html, display-name is a "phrase"
    if (/["<>)(;\\@:]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name is not valid');

    return null;
}

function validateOwnerType(type) {
    const OWNERTYPES = [ OWNERTYPE_USER, OWNERTYPE_GROUP, OWNERTYPE_APP ];
    return OWNERTYPES.includes(type);
}

async function getDomain(domain) {
    assert.strictEqual(typeof domain, 'string');

    const result = await database.query(`SELECT ${MAILDB_FIELDS} FROM mail WHERE domain = ?`, [ domain ]);
    if (result.length === 0) return null;
    return postProcessDomain(result[0]);
}

async function updateDomain(domain, data) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');

    const args = [ ];
    const fields = [ ];
    for (const k in data) {
        if (k === 'catchAll' || k === 'banner') {
            fields.push(`${k}Json = ?`);
            args.push(JSON.stringify(data[k]));
        } else if (k === 'relay') {
            fields.push('relayJson = ?');
            args.push(JSON.stringify(data[k]));
        } else {
            fields.push(k + ' = ?');
            args.push(data[k]);
        }
    }
    args.push(domain);

    const result = await database.query('UPDATE mail SET ' + fields.join(', ') + ' WHERE domain=?', args);
    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Mail domain not found');
}

async function listDomains() {
    const results = await database.query(`SELECT ${MAILDB_FIELDS} FROM mail ORDER BY domain`);
    results.forEach(function (result) { postProcessDomain(result); });
    return results;
}

async function checkOutboundPort25(family) {
    const ip = family === 4 ? await network.getIPv4() : await network.getIPv6();
    if (ip === null) return; // ipv4/ipv6 is disabled

    return await new Promise((resolve, reject) => {
        const client = new net.Socket();
        client.setTimeout(5000);
        client.connect({ port: 25, host: constants.PORT25_CHECK_SERVER, family }); // family is 4 to keep it predictable
        client.on('connect', function () {
            client.destroy(); // do not use end() because it still triggers timeout
            resolve();
        });
        client.on('timeout', function () {
            client.destroy();
            reject(new Error(`IPv${family} connect to ${constants.PORT25_CHECK_SERVER} timed out`));
        });
        client.on('error', function (error) {
            client.destroy();
            reject(new Error(`IPv${family} connect to ${constants.PORT25_CHECK_SERVER} failed: ${error.message}`));
        });
    });
}

async function checkSmtpRelay(relay) {
    assert.strictEqual(typeof relay, 'object');

    if (relay.provider === 'noop') return { status: 'skipped', message: 'Outbound disabled' };
    if (relay.provider === 'cloudron-smtp') {
        const results = await Promise.allSettled([ checkOutboundPort25(4), checkOutboundPort25(6) ]);
        if (results[0].status === 'fulfilled' && results[1].status === 'fulfilled') return { status: 'passed', message: 'Port 25 outbound is unblocked' };
        if (results[0].status === 'fulfilled') return { status: 'passed', message: 'IPv4 port 25 outbound is unblocked. IPv6 port 25 outbound is blocked and delivery to IPv6 only servers will fail' }; // ipv6 only servers are not really common
        if (results[1].status === 'fulfilled') return { status: 'failed', message: `IPv4 port 25 outbound is blocked: ${results[0].reason.message}` }; // only IPv6 worked
        return { status: 'failed', message: `Port 25 outbound is blocked. ${results[0].reason.message}` }; // both failed
    }

    const options = {
        connectionTimeout: 5000,
        greetingTimeout: 5000,
        host: relay.host,
        port: relay.port,
        secure: false, // true is for implicit TLS, false is for maybe STARTTLS
        requireTLS: true // force STARTTLS . haraka only supports STARTTLS in outbound plugin
    };

    // only set auth if either username or password is provided, some relays auth based on IP (range)
    if (relay.username || relay.password) {
        options.auth = {
            user: relay.username,
            pass: relay.password
        };
    }

    if (relay.acceptSelfSignedCerts) options.tls = { rejectUnauthorized: false };

    const transporter = nodemailer.createTransport(options);

    const [error] = await safe(transporter.verify());
    const result = {
        status: error ? 'failed' : 'passed',
        message: error ? error.message : `Connection to ${relay.host}:${relay.port} succeeded`
    };

    return result;
}

function txtToDict(txt) {
    const dict = {};
    txt.split(';').forEach(function(v) {
        const p = v.trim().split('=');
        dict[p[0]]=p[1];
    });
    return dict;
}

async function checkDkim(mailDomain) {
    assert.strictEqual(typeof mailDomain, 'object');

    if (mailDomain.relay.provider === 'noop') return { status: 'skipped', message: 'Outbound disabled' };
    if (mailDomain.relay.provider !== 'cloudron-smtp') return { status: 'skipped', message: 'DKIM check skipped, email is sent through a relay service' };

    const { domain } = mailDomain;

    const result = {
        domain: `${mailDomain.dkimSelector}._domainkey.${domain}`,
        name: `${mailDomain.dkimSelector}._domainkey`,
        type: 'TXT',
        expected: null,
        value: null,
        status: 'failed',
        message: ''
    };

    const publicKey = mailDomain.dkimKey.publicKey.split('\n').slice(1, -2).join(''); // remove header, footer and new lines

    result.expected = `v=DKIM1; t=s; p=${publicKey}`;

    const [error, txtRecords] = await safe(dig.resolve(result.domain, result.type, DNS_OPTIONS));
    if (error) return Object.assign(result, { status: 'failed', message: error.message });
    if (txtRecords.length === 0) return Object.assign(result, { status: 'failed', message: 'No DKIM record' });

    result.value = txtRecords[0].join('');
    const actual = txtToDict(result.value);
    result.status = actual.p === publicKey ? 'passed' : 'failed';

    return result;
}

async function checkSpf(mailDomain, mailFqdn) {
    assert.strictEqual(typeof mailDomain, 'object');
    assert.strictEqual(typeof mailFqdn, 'string');

    if (mailDomain.relay.provider === 'noop') return { status: 'skipped', message: 'Outbound disabled' };
    if (mailDomain.relay.provider !== 'cloudron-smtp') return { status: 'skipped', message: 'SPF check skipped. Please check that the relay provider has correct SPF settings for this domain' };

    const result = {
        domain: mailDomain.domain,
        name: '@',
        type: 'TXT',
        value: null,
        expected: `v=spf1 a:${mailFqdn} ~all`,
        status: 'failed',
        message: ''
    };

    const [error, txtRecords] = await safe(dig.resolve(result.domain, result.type, DNS_OPTIONS));
    if (error) return Object.assign(result, { status: 'failed', message: error.message });

    let i;
    for (i = 0; i < txtRecords.length; i++) {
        const txtRecord = txtRecords[i].join(''); // https://agari.zendesk.com/hc/en-us/articles/202952749-How-long-can-my-SPF-record-be-
        if (txtRecord.indexOf('v=spf1 ') !== 0) continue; // not SPF
        result.value = txtRecord;
        result.status = result.value.indexOf(` a:${mailFqdn}`) !== -1 ? 'passed' : 'failed';
        break;
    }

    if (result.status === 'passed') {
        result.expected = result.value;
    } else if (i !== txtRecords.length) {
        result.expected = `v=spf1 a:${mailFqdn} ` + result.value.slice('v=spf1 '.length);
    }

    return result;
}

async function checkMx(mailDomain, mailFqdn) {
    assert.strictEqual(typeof mailDomain, 'object');
    assert.strictEqual(typeof mailFqdn, 'string');

    if (!mailDomain.enabled) return { status: 'skipped', message: 'MX check skipped, server does not handle incoming email for this domain' };

    const { domain } = mailDomain;

    const result = {
        domain,
        name: '@',
        type: 'MX',
        value: null,
        expected: `10 ${mailFqdn}.`,
        status: 'failed',
        message: ''
    };

    const [error, mxRecords] = await safe(dig.resolve(result.domain, result.type, DNS_OPTIONS));
    if (error) return Object.assign(result, { status: 'failed', message: error.message });
    if (mxRecords.length === 0) return Object.assign(result, { status: 'failed', message: 'No MX record' });

    result.status = mxRecords.some(mx => mx.exchange === mailFqdn) ? 'passed' : 'failed'; // this lets use change priority and/or setup backup MX
    result.value = mxRecords.map(function (r) { return r.priority + ' ' + r.exchange + '.'; }).join(' ');

    if (result.status === 'passed') return result; // MX record is "my."

    // cloudflare might create a conflict subdomain (https://support.cloudflare.com/hc/en-us/articles/360020296512-DNS-Troubleshooting-FAQ)
    const [error2, mxIps] = await safe(dig.resolve(mxRecords[0].exchange, 'A', DNS_OPTIONS));
    if (error2 || mxIps.length !== 1) return result;

    const [error3, ip] = await safe(network.getIPv4());
    if (error3) return result;

    result.status = mxIps[0] === ip ? 'passed' : 'failed';

    return result;
}

async function checkDmarc(mailDomain) {
    assert.strictEqual(typeof mailDomain, 'object');

    if (!mailDomain.enabled) return { status: 'skipped', message: 'DMARC check skipped, server does not handle incoming email for this domain' };

    const { domain } = mailDomain;

    const result = {
        domain: `_dmarc.${domain}`,
        name: '_dmarc',
        type: 'TXT',
        value: null,
        expected: 'v=DMARC1; p=reject; pct=100',
        status: 'failed',
        message: ''
    };

    const [error, txtRecords] = await safe(dig.resolve(result.domain, result.type, DNS_OPTIONS));
    if (error) return Object.assign(result, { status: 'failed', message: error.message });
    if (txtRecords.length === 0) return Object.assign(result, { status: 'failed', message: 'No DMARC record' });

    result.value = txtRecords[0].join('');
    const actual = txtToDict(result.value);
    result.status = actual.v === 'DMARC1' ? 'passed' : 'failed'; // see box#666

    return result;
}

function reverseIPv6(ipv6) {
    const parts = ipv6.split('::');
    const left = parts[0].split(':');
    const right = parts[1] ? parts[1].split(':') : [];
    const fill = new Array(8 - left.length - right.length).fill('0');
    const full = [...left, ...fill, ...right];
    const expanded = full.map(part => part.padStart(4, '0')).join('');
    const reversed = expanded.split('').reverse().join('');
    const reversedWithDots = reversed.split('').join('.');
    return reversedWithDots;
}

async function checkPtr6(mailDomain, mailFqdn) {
    assert.strictEqual(typeof mailDomain, 'object');
    assert.strictEqual(typeof mailFqdn, 'string');

    if (mailDomain.relay.provider === 'noop') return { status: 'skipped', message: 'Outbound disabled' };
    if (mailDomain.relay.provider !== 'cloudron-smtp') return { status: 'skipped', message: 'PTR6 check was skipped, email is sent through a relay service' };

    const result = {
        domain: null,
        name: null,
        type: 'PTR',
        value: null,
        expected: mailFqdn, // any trailing '.' is added by client software (https://lists.gt.net/spf/devel/7918)
        status: 'failed',
        message: ''
    };

    const [error, ip] = await safe(network.getIPv6());
    if (error) return Object.assign(result, { status: 'failed', message: error.message });
    if (ip === null) return Object.assign(result, { status: 'skipped', message: 'PTR6 check was skipped, server has no IPv6' });

    const reversed = reverseIPv6(ip);
    result.domain = `${reversed}.ip6.arpa`;
    result.name = ip;

    const [error2, ptrRecords] = await safe(dig.resolve(result.domain, 'PTR', DNS_OPTIONS));
    if (error2) return Object.assign(result, { status: 'failed', message: error2.message });
    if (ptrRecords.length === 0) return Object.assign(result, { status: 'failed', message: 'No PTR6 record' });

    result.value = ptrRecords.join(' ');
    result.status = ptrRecords.some(function (v) { return v === result.expected; }) ? 'passed' : 'failed';

    return result;
}

async function checkPtr4(mailDomain, mailFqdn) {
    assert.strictEqual(typeof mailDomain, 'object');
    assert.strictEqual(typeof mailFqdn, 'string');

    if (mailDomain.relay.provider === 'noop') return { status: 'skipped', message: 'Outbound disabled' };
    if (mailDomain.relay.provider !== 'cloudron-smtp') return { status: 'skipped', message: 'PTR4 check was skipped, email is sent through a relay service' };

    const result = {
        domain: null,
        name: null,
        type: 'PTR',
        value: null,
        expected: mailFqdn, // any trailing '.' is added by client software (https://lists.gt.net/spf/devel/7918)
        status: 'failed',
        message: ''
    };

    const [error, ip] = await safe(network.getIPv4());
    if (error) return Object.assign(result, { status: 'failed', message: error.message });
    if (ip === null) return Object.assign(result, { status: 'skipped', message: 'PTR4 check was skipped, server has no IPv4' });

    result.domain = ip.split('.').reverse().join('.') + '.in-addr.arpa';
    result.name = ip;

    const [error2, ptrRecords] = await safe(dig.resolve(result.domain, 'PTR', DNS_OPTIONS));
    if (error2) return Object.assign(result, { status: 'failed', message: error2.message });
    if (ptrRecords.length === 0) return Object.assign(result, { status: 'failed', message: 'No PTR4 record' });

    result.value = ptrRecords.join(' ');
    result.status = ptrRecords.some(function (v) { return v === result.expected; }) ? 'passed' : 'failed';

    return result;
}

// https://raw.githubusercontent.com/jawsome/node-dnsbl/master/list.json https://multirbl.valli.org/list/
const RBL_LIST = [
    {
        name: 'Barracuda',
        dns: 'b.barracudacentral.org',
        site: 'https://barracudacentral.org/',
        removal: 'http://www.barracudacentral.org/rbl/removal-request',
    },
    {
        name: 'Multi SURBL',
        dns: 'multi.surbl.org',
        site: 'http://www.surbl.org',
        removal: 'https://surbl.org/surbl-analysis',
    },
    {
        name: 'Passive Spam Block List',
        dns: 'psbl.surriel.com',
        site: 'https://psbl.org',
        removal: 'https://psbl.org',
    },
    {
        name: 'SpamCop',
        dns: 'bl.spamcop.net',
        site: 'http://spamcop.net',
        removal: 'https://www.spamcop.net/bl.shtml',
    },
    {
        name: 'SpamHaus Zen',
        dns: 'zen.spamhaus.org',
        site: 'https://www.spamhaus.org/blocklists/zen-blocklist/',
        removal: 'https://check.spamhaus.org/',
        ipv6: true
    },
    {
        name: 'The Unsubscribe Blacklist(UBL)',
        dns: 'ubl.unsubscore.com ',
        site: 'https://blacklist.lashback.com/',
        removal: 'https://blacklist.lashback.com/',
    },
    {
        name: 'UCEPROTECT Network',
        dns: 'dnsbl-1.uceprotect.net', // it has 3 "zones"
        site: 'http://www.uceprotect.net/en',
        removal: 'https://www.uceprotect.net/en/index.php?m=7&s=0',
    }
];

// https://tools.ietf.org/html/rfc5782
async function checkRbl(type, mailDomain) {
    assert.strictEqual(typeof type, 'string');
    assert.strictEqual(typeof mailDomain, 'object');

    if (mailDomain.relay.provider === 'noop') return { status: 'skipped', message: 'Outbound disabled' };
    if (mailDomain.relay.provider !== 'cloudron-smtp') return { status: 'skipped', message: 'RBL check was skipped, email is sent through a relay service' };

    const { domain } = mailDomain;

    const [error, ip] = await safe(type === 'ipv4' ? network.getIPv4() : network.getIPv6());
    if (error) return { status: 'failed', ip: null, servers: [], message: `Unable to determine server ${type}: ${error.message}` };
    if (ip === null) return { status: 'skipped', ip: null, servers: [], message: `RBL check was skipped, server has no ${type}` };

    const flippedIp = type === 'ipv4' ? ip.split('.').reverse().join('.') : reverseIPv6(ip);

    const blockedServers = [];
    for (const rblServer of RBL_LIST) {
        if (type === 'ipv6' && rblServer[type] !== true) continue; // all support ipv4

        const [rblError, records] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'A', DNS_OPTIONS));
        if (rblError || records.length === 0) continue; // not listed

        log(`checkRbl (${domain}) flippedIp: ${flippedIp} is in the blocklist of ${rblServer.dns}: ${JSON.stringify(records)}`);

        const result = Object.assign({}, rblServer);

        const [error2, txtRecords] = await safe(dig.resolve(`${flippedIp}.${rblServer.dns}`, 'TXT', DNS_OPTIONS));
        result.txtRecords = error2 || !txtRecords ? [] : txtRecords.map(x => x.join(''));

        log(`checkRbl (${domain}) error: ${error2?.message || null} txtRecords: ${JSON.stringify(txtRecords)}`);

        blockedServers.push(result);
    }

    return {
        status: blockedServers.length === 0 ? 'passed' : 'failed',
        ip,
        servers: blockedServers,
        message: ''
    };
}

async function getStatus(domain) {
    assert.strictEqual(typeof domain, 'string');

    const { fqdn } = await mailServer.getLocation();

    const mailDomain = await getDomain(domain);
    if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    // mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type, status } }
    // rbl4/rbl6: { status, ip, servers: [{name,site,dns}]}
    // relay: { status, message } always checked
    const results = {};

    const checks = [
        { what: 'mx', promise: checkMx(mailDomain, fqdn) },
        { what: 'dmarc', promise: checkDmarc(mailDomain) },
        { what: 'spf', promise: checkSpf(mailDomain, fqdn) },
        { what: 'dkim', promise: checkDkim(mailDomain) },
        { what: 'ptr4', promise: checkPtr4(mailDomain, fqdn) },
        { what: 'ptr6', promise: checkPtr6(mailDomain, fqdn) },
        { what: 'rbl4', promise: checkRbl('ipv4', mailDomain) },
        { what: 'rbl6', promise: checkRbl('ipv6', mailDomain) },
        { what: 'relay', promise: checkSmtpRelay(mailDomain.relay) }
    ];

    const responses = await Promise.allSettled(checks.map(c => c.promise)); // wait for all the checks and record the result
    for (let i = 0; i < checks.length; i++) {
        const response = responses[i], check = checks[i];
        if (response.status !== 'fulfilled') {
            log(`check ${check.what} was rejected. This is not expected. reason: ${response.reason}`);
            continue;
        }

        if (response.value.message) log(`${check.what} (${domain}): ${response.value.message}`);
        safe.set(results, checks[i].what, response.value || {});
    }

    return results;
}

async function checkConfiguration() {
    const messages = {};

    const allDomains = await listDomains();

    for (const domainObject of allDomains) {
        const result = await getStatus(domainObject.domain);

        const message = [];

        [ 'mx', 'dmarc', 'spf', 'dkim', 'ptr4', 'ptr6' ].forEach((type) => {
            const record = result[type];
            if (record.status === 'failed') message.push(`${type.toUpperCase()} DNS record (${record.type}) did not match.\n    * Hostname: \`${record.name}\`\n    * Expected: \`${record.expected}\`\n    * Actual: \`${record.value || record.message}\``);
        });
        if (result.relay.status === 'failed') message.push(`Relay error: ${result.relay.message}`);
        if (result.rbl4.status === 'failed') {
            const servers = result.rbl.servers.map((bs) => `[${bs.name}](${bs.site})`); // in markdown
            message.push(`This server's IP \`${result.rbl.ip}\` is blocked in the following servers - ${servers.join(', ')}`);
        }

        if (message.length) messages[domainObject.domain] = message;
    }

    // create bulleted list for each domain
    let markdownMessage = '';
    Object.keys(messages).forEach((domain) => {
        markdownMessage += `**${domain}**\n`;
        markdownMessage += messages[domain].map((m) => `* ${m}\n`).join('');
        markdownMessage += '\n\n';
    });

    if (markdownMessage) markdownMessage += 'See [Cloudron Documentaion - Email PTR record](https://docs.cloudron.io/email#ptr-record) for more information.\n';

    return { status: markdownMessage === '', message: markdownMessage };
}

// https://agari.zendesk.com/hc/en-us/articles/202952749-How-long-can-my-SPF-record-be-
async function txtRecordsWithSpf(domain, mailFqdn) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');

    const txtRecords = await dns.getDnsRecords('', domain, 'TXT');

    log('txtRecordsWithSpf: current txt records - %j', txtRecords);

    let i, matches, validSpf;

    for (i = 0; i < txtRecords.length; i++) {
        matches = txtRecords[i].match(/^("?v=spf1) /); // DO backend may return without quotes
        if (matches === null) continue;

        // this won't work if the entry is arbitrarily "split" across quoted strings
        validSpf = txtRecords[i].indexOf('a:' + mailFqdn) !== -1;
        break; // there can only be one SPF record
    }

    if (validSpf) return null;

    if (!matches) { // no spf record was found, create one
        txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"');
        log('txtRecordsWithSpf: adding txt record');
    } else { // just add ourself
        txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length);
        log('txtRecordsWithSpf: inserting txt record');
    }

    return txtRecords;
}

async function upsertDnsRecords(domain, mailFqdn) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');

    log(`upsertDnsRecords: updating mail dns records domain:${domain} mailFqdn:${mailFqdn}`);

    const mailDomain = await getDomain(domain);
    if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    if (constants.TEST) return;

    const publicKey = mailDomain.dkimKey.publicKey.split('\n').slice(1, -2).join(''); // remove header, footer and new lines

    // t=s limits the domainkey to this domain and not it's subdomains
    const dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain, type: 'TXT', values: [ `"v=DKIM1; t=s; p=${publicKey}"` ] };

    const records = [];
    records.push(dkimRecord);
    if (mailDomain.enabled) records.push({ subdomain: '', domain, type: 'MX', values: [ '10 ' + mailFqdn + '.' ] });

    const txtRecords = await txtRecordsWithSpf(domain, mailFqdn);
    if (txtRecords) records.push({ subdomain: '', domain, type: 'TXT', values: txtRecords });

    const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting
    if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });

    log(`upsertDnsRecords: updating ${domain} with ${records.length} records: ${JSON.stringify(records)}`);

    for (const record of records) {
        await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values);
    }

    log(`upsertDnsRecords: records of ${domain} added`);
}

async function setDnsRecords(domain) {
    assert.strictEqual(typeof domain, 'string');

    const { fqdn } = await mailServer.getLocation();
    await upsertDnsRecords(domain, fqdn);
}

async function clearDomains() {
    await database.query('DELETE FROM mail', []);
}

function removePrivateFields(domain) {
    const result = _.pick(domain, ['domain', 'enabled', 'mailFromValidation', 'catchAll', 'relay', 'banner']);
    if ('password' in result.relay) {
        if ('username' in result.relay && result.relay.username === result.relay.password) delete result.relay.username;
        delete result.relay.password;
    }
    return result;
}

async function setMailFromValidation(domain, enabled) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof enabled, 'boolean');

    await updateDomain(domain, { mailFromValidation: enabled });

    safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}

async function setBanner(domain, banner) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof banner, 'object');

    await updateDomain(domain, { banner });

    safe(mailServer.restart(), { debug: log });
}

async function setCatchAllAddress(domain, addresses) {
    assert.strictEqual(typeof domain, 'string');
    assert(Array.isArray(addresses));

    for (const address of addresses) {
        if (!validator.isEmail(address)) throw new BoxError(BoxError.BAD_FIELD, `Invalid catch all address: ${address}`);
    }

    await updateDomain(domain, { catchAll: addresses });

    safe(mailServer.restart(), { debug: log }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}

async function setMailRelay(domain, relay, options) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof relay, 'object');
    assert.strictEqual(typeof options, 'object');

    const result = await getDomain(domain);
    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'Mail domain not found');

    // inject current username/password
    if ('password' in result.relay) {
        if (!Object.hasOwn(relay, 'username')) relay.username = result.relay.username;
        if (!Object.hasOwn(relay, 'password')) relay.password = result.relay.password;
    }

    if (!options.skipVerify) {
        const relayResult = await checkSmtpRelay(relay);
        if (relayResult.status === 'failed') throw new BoxError(BoxError.BAD_FIELD, relayResult.message);
    }

    await updateDomain(domain, { relay });

    safe(mailServer.restart(), { debug: log });
}

async function setMailEnabled(domain, enabled, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof enabled, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');

    await updateDomain(domain, { enabled });

    await mailServer.restart();
    await platform.onMailServerIncomingDomainsChanged(auditSource);

    await eventlog.add(enabled ? eventlog.ACTION_MAIL_ENABLED : eventlog.ACTION_MAIL_DISABLED, auditSource, { domain });
}

async function sendTestMail(domain, to) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof to, 'string');

    const result = await getDomain(domain);
    if (!result) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    await mailer.sendTestMail(result.domain, to);
}

async function listMailboxesByDomain(domain, page, perPage) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    // const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
    // const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use

    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3, m1.storageQuota AS storageQuota, m1.messagesQuota AS messagesQuota '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
        + ' WHERE m1.domain = ?'
        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
        // + searchQuery
        + ' ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ domain, (page-1)*perPage, perPage ]);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function listMailboxes(page, perPage) {
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3, m1.storageQuota AS storageQuota, m1.messagesQuota AS messagesQuota '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
        + ' ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ (page-1)*perPage, perPage ]);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function listMailboxesByUserId(userId) {
    assert.strictEqual(typeof userId, 'string');

    const groupIds = await groups._getMembership(userId);

    const baseQuery = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3, m1.storageQuota AS storageQuota, m1.messagesQuota AS messagesQuota '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId';

    let whereClause = " WHERE (m1.ownerType = 'user' AND m1.ownerId = ?)";
    const args = [ userId ];
    if (groupIds.length > 0) {
        const placeholders = groupIds.map(() => '?').join(',');
        whereClause += ` OR (m1.ownerType = '${OWNERTYPE_GROUP}' AND m1.ownerId IN (${placeholders}))`;
        args.push(...groupIds);
    }

    const query = baseQuery + whereClause + ' GROUP BY m1.name, m1.domain, m1.ownerId ORDER BY name';

    const results = await database.query(query, args);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function delByDomain(domain) {
    assert.strictEqual(typeof domain, 'string');

    await database.query('DELETE FROM mailboxes WHERE domain = ?', [ domain ]);
}

async function get(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE name = ? AND domain = ?`, [ name, domain ]);
    if (results.length === 0) return null;

    return postProcessMailbox(results[0]);
}

async function getMailbox(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE name = ? AND type = ? AND domain = ?`, [ name, TYPE_MAILBOX, domain ]);
    if (results.length === 0) return null;
    return postProcessMailbox(results[0]);
}

async function addMailbox(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { ownerId, ownerType, active, storageQuota, messagesQuota, enablePop3 } = data;
    assert.strictEqual(typeof ownerId, 'string');
    assert.strictEqual(typeof ownerType, 'string');
    assert.strictEqual(typeof active, 'boolean');
    assert.strictEqual(typeof enablePop3, 'boolean');
    assert(Number.isInteger(storageQuota) && storageQuota >= 0);
    assert(Number.isInteger(messagesQuota) && messagesQuota >= 0);

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    if (!validateOwnerType(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');

    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active, storageQuota, messagesQuota, enablePop3) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
        [ name, TYPE_MAILBOX, domain, ownerId, ownerType, active, storageQuota, messagesQuota, enablePop3 ]));
    if (error && error.sqlCode === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
    if (error && error.sqlCode === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('mailboxes_domain_constraint')) throw new BoxError(BoxError.NOT_FOUND, `no such domain '${domain}'`);
    if (error) throw error;

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active, storageQuota, enablePop3, messageQuota: messagesQuota });
}

async function updateMailbox(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const args = [];
    const fields = [];
    for (const k in data) {
        if (k === 'enablePop3' || k === 'active') {
            fields.push(k + ' = ?');
            args.push(data[k] ? 1 : 0);
            continue;
        }

        if (k === 'ownerType' && !validateOwnerType(data[k])) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');

        fields.push(k + ' = ?');
        args.push(data[k]);
    }
    args.push(name.toLowerCase());
    args.push(domain);

    const mailbox = await getMailbox(name, domain);
    if (!mailbox) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');

    const result = await safe(database.query('UPDATE mailboxes SET ' + fields.join(', ') + ' WHERE name = ? AND domain = ?', args));
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, Object.assign(data, { name, domain, oldUserId: mailbox.userId }) );
}

async function removeSolrIndex(mailbox) {
    assert.strictEqual(typeof mailbox, 'string');

    const addonDetails = await services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN');

    const [error, response] = await safe(superagent.post(`http://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`)
        .timeout(2000)
        .send({ mailbox })
        .ok(() => true));

    if (error) throw new BoxError(BoxError.MAIL_ERROR, `Could not remove solr index: ${error.message}`);

    if (response.status !== 200) throw new BoxError(BoxError.MAIL_ERROR, `Error removing solr index - ${response.status} ${response.text}`);
}

async function delMailbox(name, domain, options, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const mailbox =`${name}@${domain}`;
    if (options.deleteMails) {
        const [error] = await safe(shell.sudo([ REMOVE_MAILBOX_CMD, mailbox ], {}));
        if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing mailbox: ${error.message}`);
    }

    // deletes aliases as well
    const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    const [error] = await safe(removeSolrIndex(mailbox));
    if (error) log(`delMailbox: failed to remove solr index: ${error.message}`);

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
}

async function getAlias(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE name = ? AND type = ? AND domain = ?`, [ name, TYPE_ALIAS, domain ]);
    if (results.length === 0) return null;

    results.forEach(function (result) { postProcessMailbox(result); });

    return results[0];
}

async function searchAlias(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE ? LIKE REPLACE(REPLACE(name, '*', '%'), '_', '\\_') AND type = ? AND domain = ?`, [ name, TYPE_ALIAS, domain ]);
    if (results.length === 0) return null;

    results.forEach(function (result) { postProcessMailbox(result); });

    return results[0];
}

async function getAliases(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const result = await getMailbox(name, domain); // check if mailbox exists
    if (result === null) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');
    return await database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name', [ TYPE_ALIAS, name, domain ]);
}

async function setAliases(name, domain, aliases, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert(Array.isArray(aliases));
    assert.strictEqual(typeof auditSource, 'object');

    for (let i = 0; i < aliases.length; i++) {
        const aliasName = aliases[i].name.toLowerCase();
        const aliasDomain = aliases[i].domain.toLowerCase();

        const error = validateAlias(aliasName);
        if (error) throw error;

        const mailDomain = await getDomain(aliasDomain);
        if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, `mail domain ${aliasDomain} not found`);

        aliases[i] = { name: aliasName, domain: aliasDomain };
    }

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
    if (results.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    const queries = [];
    // clear existing aliases
    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, TYPE_ALIAS ] });
    for (const alias of aliases) {
        queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
            args: [ alias.name, alias.domain, TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
    }

    const [error] = await safe(database.transaction(queries));
    if (error && error.sqlCode === 'ER_DUP_ENTRY' && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
        const aliasMatch = error.message.match(new RegExp(`Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'`));
        if (!aliasMatch) throw new BoxError(BoxError.ALREADY_EXISTS, error.message);

        throw new BoxError(BoxError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`);
    }
    if (error) throw error;

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, aliases });
}

async function listMailingListsByDomain(domain, page, perPage) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
    // if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ' OR membersJson LIKE ' + mysql.escape('%' + search + '%') + ')';

    query += 'ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ TYPE_LIST, domain, (page-1)*perPage, perPage ]);

    results.forEach(function (result) { postProcessMailbox(result); });

    return results;
}

async function getStats(domain) {
    assert.strictEqual(typeof domain, 'string');

    const mailboxes = await listMailboxesByDomain(domain, 1, 10000);
    const mailingLists = await listMailingListsByDomain(domain, 1, 10000);

    return {
        mailboxCount: mailboxes.length,
        pop3Count: mailboxes.filter(mb => mb.enablePop3).length,
        aliasCount: mailboxes.map(mb => mb.aliases.length).reduce((a, b) => a + b, 0),
        mailingListCount: mailingLists.length
    };
}

async function getMailingList(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND name = ? AND domain = ?', [ TYPE_LIST, name, domain ]);
    if (results.length === 0) return null;

    return postProcessMailbox(results[0]);
}

async function addMailingList(name, domain, data, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { members, membersOnly, active } = data;
    assert(Array.isArray(members));
    assert.strictEqual(typeof membersOnly, 'boolean');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    for (let i = 0; i < members.length; i++) {
        if (!validator.isEmail(members[i])) throw new BoxError(BoxError.BAD_FIELD, 'Invalid mail member: ' + members[i]);
    }

    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, membersJson, membersOnly, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', [ name, TYPE_LIST, domain, 'admin', 'user', JSON.stringify(members), membersOnly, active ]));
    if (error && error.sqlCode === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
    if (error) throw error;

    await eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
}

async function updateMailingList(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { members, membersOnly, active } = data;
    assert(Array.isArray(members));
    assert.strictEqual(typeof membersOnly, 'boolean');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    const error = validateName(name);
    if (error) throw error;

    for (let i = 0; i < members.length; i++) {
        if (!validator.isEmail(members[i])) throw new BoxError(BoxError.BAD_FIELD, 'Invalid email: ' + members[i]);
    }

    const result = await database.query('UPDATE mailboxes SET membersJson = ?, membersOnly = ?, active = ? WHERE name = ? AND domain = ?',
        [ JSON.stringify(members), membersOnly, active, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    await eventlog.add(eventlog.ACTION_MAIL_LIST_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
}

async function delMailingList(name, domain, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    // deletes aliases as well
    const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    await eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
}

// resolves the members of a list. i.e the lists and aliases
async function resolveMailingList(listName, listDomain) {
    assert.strictEqual(typeof listName, 'string');
    assert.strictEqual(typeof listDomain, 'string');

    const mailDomains = await listDomains();
    const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');

    const list = await getMailingList(listName, listDomain);
    if (!list) throw new BoxError(BoxError.NOT_FOUND, 'List not found');

    const resolvedMembers = [], visited = []; // slice creates a copy of array
    let toResolve = list.members.slice();

    while (toResolve.length != 0) {
        const toProcess = toResolve.shift();
        const parts = toProcess.split('@');
        const memberName = parts[0].split('+')[0], memberDomain = parts[1];

        if (!mailInDomains.includes(memberDomain)) {  // external domain
            resolvedMembers.push(toProcess);
            continue;
        }

        const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress
        if (visited.includes(member)) {
            log(`resolveMailingList: list ${listName}@${listDomain} has a recursion at member ${member}`);
            continue;
        }
        visited.push(member);

        const entry = await get(memberName, memberDomain);
        if (!entry) { // let it bounce
            resolvedMembers.push(member);
            continue;
        }

        if (entry.type === TYPE_MAILBOX) { // concrete mailbox
            resolvedMembers.push(member);
        } else if (entry.type === TYPE_ALIAS) { // resolve aliases
            toResolve = toResolve.concat(`${entry.aliasName}@${entry.aliasDomain}`);
        } else { // resolve list members
            toResolve = toResolve.concat(entry.members);
        }
    }

    return { resolvedMembers, list };
}

async function checkStatus() {
    const result = await checkConfiguration();
    if (result.status) {
        await notifications.unpin(notifications.TYPE_MAIL_STATUS, {});
    } else {
        await notifications.pin(notifications.TYPE_MAIL_STATUS, 'Email is not configured properly', result.message, {});
    }
}

const _delByDomain = delByDomain;

export default {
    getStatus,
    checkConfiguration,
    listDomains,
    getDomain,
    clearDomains,
    removePrivateFields,
    setDnsRecords,
    upsertDnsRecords,
    validateName,
    validateDisplayName,
    setMailFromValidation,
    setCatchAllAddress,
    setMailRelay,
    setMailEnabled,
    setBanner,
    sendTestMail,
    listMailboxesByDomain,
    listMailboxes,
    listMailboxesByUserId,
    getMailbox,
    addMailbox,
    updateMailbox,
    delMailbox,
    getAlias,
    getAliases,
    setAliases,
    searchAlias,
    listMailingListsByDomain,
    getMailingList,
    addMailingList,
    updateMailingList,
    delMailingList,
    resolveMailingList,
    getStats,
    checkStatus,
    OWNERTYPE_USER,
    OWNERTYPE_GROUP,
    OWNERTYPE_APP,
    TYPE_MAILBOX,
    TYPE_LIST,
    TYPE_ALIAS,
    _delByDomain,
    _updateDomain: updateDomain,
};