cloudron-box/src/mail.js

'use strict';

exports = module.exports = {
    getStatus,
    checkConfiguration,

    getLocation,
    setLocation, // triggers the change task
    changeLocation, // does the actual changing

    listDomains,

    getDomain,
    clearDomains,

    onDomainAdded,
    onDomainRemoved,

    removePrivateFields,

    setDnsRecords,

    validateName,

    setMailFromValidation,
    setCatchAllAddress,
    setMailRelay,
    setMailEnabled,
    setBanner,

    startMail: restartMail,
    restartMail,
    handleCertChanged,
    getMailAuth,

    sendTestMail,

    getMailboxCount,
    listMailboxes,
    listAllMailboxes,
    getMailbox,
    addMailbox,
    updateMailbox,
    delMailbox,

    getAlias,
    getAliases,
    setAliases,

    getLists,
    getList,
    addList,
    updateList,
    delList,
    resolveList,

    OWNERTYPE_USER: 'user',
    OWNERTYPE_GROUP: 'group',

    DEFAULT_MEMORY_LIMIT: 512 * 1024 * 1024,

    TYPE_MAILBOX: 'mailbox',
    TYPE_LIST: 'list',
    TYPE_ALIAS: 'alias',

    _delByDomain: delByDomain,
    _readDkimPublicKeySync: readDkimPublicKeySync,
    _updateDomain: updateDomain
};

const assert = require('assert'),
    async = require('async'),
    BoxError = require('./boxerror.js'),
    cloudron = require('./cloudron.js'),
    constants = require('./constants.js'),
    database = require('./database.js'),
    debug = require('debug')('box:mail'),
    dns = require('./dns.js'),
    docker = require('./docker.js'),
    domains = require('./domains.js'),
    eventlog = require('./eventlog.js'),
    hat = require('./hat.js'),
    infra = require('./infra_version.js'),
    mailer = require('./mailer.js'),
    mysql = require('mysql'),
    net = require('net'),
    nodemailer = require('nodemailer'),
    path = require('path'),
    paths = require('./paths.js'),
    request = require('request'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    smtpTransport = require('nodemailer-smtp-transport'),
    sysinfo = require('./sysinfo.js'),
    system = require('./system.js'),
    tasks = require('./tasks.js'),
    users = require('./users.js'),
    util = require('util'),
    validator = require('validator'),
    _ = require('underscore');

const DNS_OPTIONS = { timeout: 5000 };
const NOOP_CALLBACK = function (error) { if (error) debug(error); };
const REMOVE_MAILBOX_CMD = path.join(__dirname, 'scripts/rmmailbox.sh');

const MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active' ].join(',');
const MAILDB_FIELDS = [ 'domain', 'enabled', 'mailFromValidation', 'catchAllJson', 'relayJson', 'dkimSelector', 'bannerJson' ].join(',');

const domainsList = util.callbackify(domains.list);

function postProcessMailbox(data) {
    data.members = safe.JSON.parse(data.membersJson) || [ ];
    delete data.membersJson;

    data.membersOnly = !!data.membersOnly;
    data.active = !!data.active;

    return data;
}

function postProcessAliases(data) {
    const aliasNames = JSON.parse(data.aliasNames), aliasDomains = JSON.parse(data.aliasDomains);
    delete data.aliasNames;
    delete data.aliasDomains;
    data.aliases = [];
    for (let i = 0; i < aliasNames.length; i++) { // NOTE: aliasNames is [ null ] when no aliases
        if (aliasNames[i]) data.aliases[i] = { name: aliasNames[i], domain: aliasDomains[i] };
    }

    return data;
}

function postProcessDomain(data) {
    data.enabled = !!data.enabled; // int to boolean
    data.mailFromValidation = !!data.mailFromValidation; // int to boolean

    data.catchAll = safe.JSON.parse(data.catchAllJson) || [ ];
    delete data.catchAllJson;

    data.relay = safe.JSON.parse(data.relayJson) || { provider: 'cloudron-smtp' };
    delete data.relayJson;

    data.banner = safe.JSON.parse(data.bannerJson) || { text: null, html: null };
    delete data.bannerJson;

    return data;
}

function validateName(name) {
    assert.strictEqual(typeof name, 'string');

    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox name must be atleast 1 char');
    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');

    // also need to consider valid LDAP characters here (e.g '+' is reserved)
    if (/[^a-zA-Z0-9.-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals and dot');

    return null;
}

function checkOutboundPort25(callback) {
    assert.strictEqual(typeof callback, 'function');

    var smtpServer = _.sample([
        'smtp.gmail.com',
        'smtp.live.com',
        'smtp.1und1.de',
    ]);

    var relay = {
        value: 'OK',
        status: false
    };

    var client = new net.Socket();
    client.setTimeout(5000);
    client.connect(25, smtpServer);
    client.on('connect', function () {
        relay.status = true;
        relay.value = 'OK';
        client.destroy(); // do not use end() because it still triggers timeout
        callback(null, relay);
    });
    client.on('timeout', function () {
        relay.status = false;
        relay.value = `Connect to ${smtpServer} timed out. Check if port 25 (outbound) is blocked`;
        client.destroy();
        callback(new BoxError(BoxError.TIMEOUT, `Connect to ${smtpServer} timed out.`), relay);
    });
    client.on('error', function (error) {
        relay.status = false;
        relay.value = `Connect to ${smtpServer} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
        client.destroy();
        callback(new BoxError(BoxError.NETWORK_ERROR, `Connect to ${smtpServer} failed.`), relay);
    });
}

function checkSmtpRelay(relay, callback) {
    var result = {
        value: 'OK',
        status: false
    };

    var options = {
        connectionTimeout: 5000,
        greetingTimeout: 5000,
        host: relay.host,
        port: relay.port
    };

    // only set auth if either username or password is provided, some relays auth based on IP (range)
    if (relay.username || relay.password) {
        options.auth = {
            user: relay.username,
            pass: relay.password
        };
    }

    if (relay.acceptSelfSignedCerts) options.tls = { rejectUnauthorized: false };

    var transporter = nodemailer.createTransport(smtpTransport(options));

    transporter.verify(function(error) {
        result.status = !error;
        if (error) {
            result.value = error.message;
            return callback(error, result);
        }

        callback(null, result);
    });
}

async function verifyRelay(relay) {
    assert.strictEqual(typeof relay, 'object');

    // we used to verify cloudron-smtp with checkOutboundPort25 but that is unreliable given that we just
    // randomly select some smtp server
    if (relay.provider === 'cloudron-smtp' || relay.provider === 'noop') return;

    const checkSmtpRelayAsync = util.promisify(checkSmtpRelay);

    const [error] = await safe(checkSmtpRelayAsync(relay));
    if (error) throw new BoxError(BoxError.BAD_FIELD, error.message);
}

function checkDkim(mailDomain, callback) {
    assert.strictEqual(typeof mailDomain, 'object');
    assert.strictEqual(typeof callback, 'function');

    const domain = mailDomain.domain;
    let dkim = {
        domain: `${mailDomain.dkimSelector}._domainkey.${domain}`,
        name: `${mailDomain.dkimSelector}._domainkey`,
        type: 'TXT',
        expected: null,
        value: null,
        status: false
    };

    var dkimKey = readDkimPublicKeySync(domain);
    if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, `Failed to read dkim public key of ${domain}`), dkim);

    dkim.expected = 'v=DKIM1; t=s; p=' + dkimKey;

    dns.resolve(dkim.domain, dkim.type, DNS_OPTIONS, function (error, txtRecords) {
        if (error) return callback(error, dkim);

        if (txtRecords.length !== 0) {
            dkim.value = txtRecords[0].join('');
            const actual = txtToDict(dkim.value);
            dkim.status = actual.p === dkimKey;
        }

        callback(null, dkim);
    });
}

function checkSpf(domain, mailFqdn, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof callback, 'function');

    var spf = {
        domain: domain,
        name: '@',
        type: 'TXT',
        value: null,
        expected: 'v=spf1 a:' + mailFqdn + ' ~all',
        status: false
    };

    dns.resolve(spf.domain, spf.type, DNS_OPTIONS, function (error, txtRecords) {
        if (error) return callback(error, spf);

        var i;
        for (i = 0; i < txtRecords.length; i++) {
            let txtRecord = txtRecords[i].join(''); // https://agari.zendesk.com/hc/en-us/articles/202952749-How-long-can-my-SPF-record-be-
            if (txtRecord.indexOf('v=spf1 ') !== 0) continue; // not SPF
            spf.value = txtRecord;
            spf.status = spf.value.indexOf(' a:' + settings.mailFqdn()) !== -1;
            break;
        }

        if (spf.status) {
            spf.expected = spf.value;
        } else if (i !== txtRecords.length) {
            spf.expected = 'v=spf1 a:' + settings.mailFqdn() + ' ' + spf.value.slice('v=spf1 '.length);
        }

        callback(null, spf);
    });
}

function checkMx(domain, mailFqdn, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof callback, 'function');

    var mx = {
        domain: domain,
        name: '@',
        type: 'MX',
        value: null,
        expected: '10 ' + mailFqdn + '.',
        status: false
    };

    dns.resolve(mx.domain, mx.type, DNS_OPTIONS, function (error, mxRecords) {
        if (error) return callback(error, mx);
        if (mxRecords.length === 0) return callback(null, mx);

        mx.status = mxRecords.some(mx => mx.exchange === mailFqdn); // this lets use change priority and/or setup backup MX
        mx.value = mxRecords.map(function (r) { return r.priority + ' ' + r.exchange + '.'; }).join(' ');

        if (mx.status) return callback(null, mx); // MX record is "my."

        // cloudflare might create a conflict subdomain (https://support.cloudflare.com/hc/en-us/articles/360020296512-DNS-Troubleshooting-FAQ)
        dns.resolve(mxRecords[0].exchange, 'A', DNS_OPTIONS, function (error, mxIps) {
            if (error || mxIps.length !== 1) return callback(null, mx);

            sysinfo.getServerIp(function (error, ip) {
                if (error) return callback(null, mx);

                mx.status = mxIps[0] === ip;

                callback(null, mx);
            });
        });
    });
}

function txtToDict(txt) {
    var dict = {};
    txt.split(';').forEach(function(v) {
        var p = v.trim().split('=');
        dict[p[0]]=p[1];
    });
    return dict;
}

function checkDmarc(domain, callback) {
    var dmarc = {
        domain: '_dmarc.' + domain,
        name: '_dmarc',
        type: 'TXT',
        value: null,
        expected: 'v=DMARC1; p=reject; pct=100',
        status: false
    };

    dns.resolve(dmarc.domain, dmarc.type, DNS_OPTIONS, function (error, txtRecords) {
        if (error) return callback(error, dmarc);

        if (txtRecords.length !== 0) {
            dmarc.value = txtRecords[0].join('');
            const actual = txtToDict(dmarc.value);
            dmarc.status = actual.v === 'DMARC1'; // see box#666
        }

        callback(null, dmarc);
    });
}

function checkPtr(mailFqdn, callback) {
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof callback, 'function');

    var ptr = {
        domain: null,
        name: null,
        type: 'PTR',
        value: null,
        expected: mailFqdn, // any trailing '.' is added by client software (https://lists.gt.net/spf/devel/7918)
        status: false
    };

    sysinfo.getServerIp(function (error, ip) {
        if (error) return callback(error, ptr);

        ptr.domain = ip.split('.').reverse().join('.') + '.in-addr.arpa';
        ptr.name = ip;

        dns.resolve(ptr.domain, 'PTR', DNS_OPTIONS, function (error, ptrRecords) {
            if (error) return callback(error, ptr);

            if (ptrRecords.length !== 0) {
                ptr.value = ptrRecords.join(' ');
                ptr.status = ptrRecords.some(function (v) { return v === ptr.expected; });
            }

            return callback(null, ptr);
        });
    });
}

// https://raw.githubusercontent.com/jawsome/node-dnsbl/master/list.json
const RBL_LIST = [
    {
        'name': 'Abuse.ch',
        'dns': 'spam.abuse.ch',
        'site': 'http://abuse.ch/'
    },

    {
        'name': 'Barracuda',
        'dns': 'b.barracudacentral.org',
        'site': 'http://www.barracudacentral.org/rbl/removal-request'
    },
    {
        'name': 'Composite Blocking List',
        'dns': 'cbl.abuseat.org',
        'site': 'http://www.abuseat.org'
    },
    {
        'name': 'Multi SURBL',
        'dns': 'multi.surbl.org',
        'site': 'http://www.surbl.org'
    },
    {
        'name': 'Passive Spam Block List',
        'dns': 'psbl.surriel.com',
        'site': 'https://psbl.org'
    },
    {
        'name': 'Sorbs Aggregate Zone',
        'dns': 'dnsbl.sorbs.net',
        'site': 'http://dnsbl.sorbs.net/'
    },
    {
        'name': 'Sorbs spam.dnsbl Zone',
        'dns': 'spam.dnsbl.sorbs.net',
        'site': 'http://sorbs.net'
    },
    {
        'name': 'SpamCop',
        'dns': 'bl.spamcop.net',
        'site': 'http://spamcop.net'
    },
    {
        'name': 'SpamHaus Zen',
        'dns': 'zen.spamhaus.org',
        'site': 'http://spamhaus.org'
    },
    {
        'name': 'The Unsubscribe Blacklist(UBL)',
        'dns': 'ubl.unsubscore.com ',
        'site': 'http://www.lashback.com/blacklist/'
    },
    {
        'name': 'UCEPROTECT Network',
        'dns': 'dnsbl-1.uceprotect.net',
        'site': 'http://www.uceprotect.net/en'
    }
];

// this function currently only looks for black lists based on IP. TODO: also look up by domain
function checkRblStatus(domain, callback) {
    assert.strictEqual(typeof callback, 'function');

    sysinfo.getServerIp(function (error, ip) {
        if (error) return callback(error, ip);

        var flippedIp = ip.split('.').reverse().join('.');

        // https://tools.ietf.org/html/rfc5782
        async.map(RBL_LIST, function (rblServer, iteratorDone) {
            dns.resolve(flippedIp + '.' + rblServer.dns, 'A', DNS_OPTIONS, function (error, records) {
                if (error || !records) return iteratorDone(null, null);    // not listed

                debug('checkRblStatus: %s (ip: %s) is in the blacklist of %j', domain, flippedIp, rblServer);

                var result = _.extend({ }, rblServer);

                dns.resolve(flippedIp + '.' + rblServer.dns, 'TXT', DNS_OPTIONS, function (error, txtRecords) {
                    result.txtRecords = error || !txtRecords ? 'No txt record' : txtRecords.map(x => x.join(''));

                    debug('checkRblStatus: %s (error: %s) (txtRecords: %j)', domain, error, txtRecords);

                    return iteratorDone(null, result);
                });
            });
        }, function (ignoredError, blacklistedServers) {
            blacklistedServers = blacklistedServers.filter(function(b) { return b !== null; });

            debug('checkRblStatus: %s (ip: %s) servers: %j', domain, ip, blacklistedServers);

            return callback(null, { status: blacklistedServers.length === 0, ip: ip, servers: blacklistedServers });
        });
    });
}

function getStatus(domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

    // ensure we always have a valid toplevel properties for the api
    var results = {
        dns: {}, // { mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type, status } }
        rbl: {}, // { status, ip, servers: [{name,site,dns}]} optional. only for cloudron-smtp
        relay: {} // { status, value } always checked
    };

    function recordResult(what, func) {
        return function (callback) {
            func(function (error, result) {
                if (error) debug(`Ignored error - ${what} : ${error.message}`);

                safe.set(results, what, result || {});

                callback();
            });
        };
    }

    const mailFqdn = settings.mailFqdn();
    const getDomainFunc = util.callbackify(getDomain);

    getDomainFunc(domain, function (error, mailDomain) {
        if (error) return callback(error);
        if (!mailDomain) return callback(new BoxError(BoxError.NOT_FOUND, 'mail domain not found'));

        let checks = [];
        if (mailDomain.enabled) {
            checks.push(
                recordResult('dns.mx', checkMx.bind(null, domain, mailFqdn)),
                recordResult('dns.dmarc', checkDmarc.bind(null, domain))
            );
        }

        if (mailDomain.relay.provider === 'cloudron-smtp') {
            // these tests currently only make sense when using Cloudron's SMTP server at this point
            checks.push(
                recordResult('dns.spf', checkSpf.bind(null, domain, mailFqdn)),
                recordResult('dns.dkim', checkDkim.bind(null, mailDomain)),
                recordResult('dns.ptr', checkPtr.bind(null, mailFqdn)),
                recordResult('relay', checkOutboundPort25),
                recordResult('rbl', checkRblStatus.bind(null, domain))
            );
        } else if (mailDomain.relay.provider !== 'noop') {
            checks.push(recordResult('relay', checkSmtpRelay.bind(null, mailDomain.relay)));
        }

        async.parallel(checks, function () {
            callback(null, results);
        });
    });
}

async function checkConfiguration() {
    let messages = {};

    const allDomains = await listDomains();

    for (const domainObject of allDomains) {
        const result = await util.promisify(getStatus)(domainObject.domain);

        let message = [];

        Object.keys(result.dns).forEach((type) => {
            const record = result.dns[type];
            if (!record.status) message.push(`${type.toUpperCase()} DNS record (${record.type}) did not match.\n    * Hostname: \`${record.name}\`\n    * Expected: \`${record.expected}\`\n    * Actual: \`${record.value}\``);
        });
        if (result.relay && result.relay.status === false) message.push(`Relay error: ${result.relay.value}`);
        if (result.rbl && result.rbl.status === false) { // rbl field contents is optional
            const servers = result.rbl.servers.map((bs) => `[${bs.name}](${bs.site})`); // in markdown
            message.push(`This server's IP \`${result.rbl.ip}\` is blacklisted in the following servers - ${servers.join(', ')}`);
        }

        if (message.length) messages[domainObject.domain] = message;
    }

    // create bulleted list for each domain
    let markdownMessage = '';
    Object.keys(messages).forEach((domain) => {
        markdownMessage += `**${domain}**\n`;
        markdownMessage += messages[domain].map((m) => `* ${m}\n`).join('');
        markdownMessage += '\n\n';
    });

    if (markdownMessage) markdownMessage += 'Email Status is checked every 30 minutes.\n See the [troubleshooting docs](https://docs.cloudron.io/troubleshooting/#mail-dns) for more information.\n';

    return markdownMessage; // empty message means all status checks succeeded
}

async function createMailConfig(mailFqdn, mailDomain) {
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof mailDomain, 'string');

    debug('createMailConfig: generating mail config');

    const mailDomains = await listDomains();

    const mailOutDomains = mailDomains.filter(d => d.relay.provider !== 'noop').map(d => d.domain).join(',');
    const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');

    // mail_domain is used for SRS
    if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
        `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\nmail_domain=${mailDomain}\n\n`, 'utf8')) {
        throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
    }

    // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
    if (!safe.fs.writeFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/smtp_forward.ini'), 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
        throw new BoxError(BoxError.FS_ERROR, `Could not create smtp forward file: ${safe.error.message}`);
    }

    // create sections for per-domain configuration
    for (const domain of mailDomains) {
        const catchAll = domain.catchAll.map(function (c) { return `${c}@${domain.domain}`; }).join(',');
        const mailFromValidation = domain.mailFromValidation;

        if (!safe.fs.appendFileSync(path.join(paths.ADDON_CONFIG_DIR, 'mail/mail.ini'),
            `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
            throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
        }

        if (!safe.fs.writeFileSync(`${paths.ADDON_CONFIG_DIR}/mail/banner/${domain.domain}.text`, domain.banner.text || '')) throw new BoxError(BoxError.FS_ERROR, `Could not create text banner file: ${safe.error.message}`);
        if (!safe.fs.writeFileSync(`${paths.ADDON_CONFIG_DIR}/mail/banner/${domain.domain}.html`, domain.banner.html || '')) throw new BoxError(BoxError.FS_ERROR, `Could not create html banner file: ${safe.error.message}`);

        const relay = domain.relay;

        const enableRelay = relay.provider !== 'cloudron-smtp' && relay.provider !== 'noop',
            host = relay.host || '',
            port = relay.port || 25,
            authType = relay.username ? 'plain' : '',
            username = relay.username || '',
            password = relay.password || '';

        if (!enableRelay) continue;

        if (!safe.fs.appendFileSync(paths.ADDON_CONFIG_DIR + '/mail/smtp_forward.ini',
            `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\n\n`, 'utf8')) {
            throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
        }
    }

    return mailInDomains.length !== 0 /* allowInbound */;
}

async function configureMail(mailFqdn, mailDomain, serviceConfig) {
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof mailDomain, 'string');
    assert.strictEqual(typeof serviceConfig, 'object');

    // mail (note: 2587 is hardcoded in mail container and app use this port)
    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
    // MAIL_DOMAIN is the domain for which this server is relaying mails
    // mail container uses /app/data for backed up data and /run for restart-able data

    const tag = infra.images.mail.tag;
    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
    const memory = system.getMemoryAllocation(memoryLimit);
    const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);

    const bundle = await reverseProxy.getCertificatePath(mailFqdn, mailDomain);

    const dhparamsFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/dhparams.pem');
    const mailCertFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_cert.pem');
    const mailKeyFilePath = path.join(paths.ADDON_CONFIG_DIR, 'mail/tls_key.pem');

    if (!safe.child_process.execSync(`cp ${paths.DHPARAMS_FILE} ${dhparamsFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not copy dhparams:' + safe.error.message);
    if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message);
    if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message);

    await shell.promises.exec('stopMail', 'docker stop mail || true');
    await shell.promises.exec('removeMail', 'docker rm -f mail || true');

    const allowInbound = await createMailConfig(mailFqdn, mailDomain);

    const ports = allowInbound ? '-p 587:2587 -p 993:9993 -p 4190:4190 -p 25:2587' : '';

    const cmd = `docker run --restart=always -d --name="mail" \
                --net cloudron \
                --net-alias mail \
                --log-driver syslog \
                --log-opt syslog-address=udp://127.0.0.1:2514 \
                --log-opt syslog-format=rfc5424 \
                --log-opt tag=mail \
                -m ${memory} \
                --memory-swap ${memoryLimit} \
                --dns 172.18.0.1 \
                --dns-search=. \
                -e CLOUDRON_MAIL_TOKEN="${cloudronToken}" \
                -e CLOUDRON_RELAY_TOKEN="${relayToken}" \
                -v "${paths.MAIL_DATA_DIR}:/app/data" \
                -v "${paths.PLATFORM_DATA_DIR}/addons/mail:/etc/mail" \
                ${ports} \
                --label isCloudronManaged=true \
                --read-only -v /run -v /tmp ${tag}`;

    await shell.promises.exec('startMail', cmd);
}

async function getMailAuth() {
    const dockerInspect = util.promisify(docker.inspect);

    const data = await dockerInspect('mail');
    const ip = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
    if (!ip) throw new BoxError(BoxError.MAIL_ERROR, 'Error querying mail server IP');

    // extract the relay token for auth
    const env = safe.query(data, 'Config.Env', null);
    if (!env) throw new BoxError(BoxError.MAIL_ERROR, 'Error getting mail env');
    const tmp = env.find(function (e) { return e.indexOf('CLOUDRON_RELAY_TOKEN') === 0; });
    if (!tmp) throw new BoxError(BoxError.MAIL_ERROR, 'Error getting CLOUDRON_RELAY_TOKEN env var');
    const relayToken = tmp.slice('CLOUDRON_RELAY_TOKEN'.length + 1); // +1 for the = sign
    if (!relayToken) throw new BoxError(BoxError.MAIL_ERROR, 'Error parsing CLOUDRON_RELAY_TOKEN');

    return {
        ip,
        port: constants.INTERNAL_SMTP_PORT,
        relayToken
    };
}

function restartMail(callback) {
    assert.strictEqual(typeof callback, 'function');

    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return callback();

    services.getServiceConfig('mail', async function (error, serviceConfig) {
        if (error) return callback(error);

        debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} dashboardDomain:${settings.dashboardDomain()}`);
        [error] = await safe(configureMail(settings.mailFqdn(), settings.dashboardDomain(), serviceConfig));
        callback(error);
    });
}

async function restartMailIfActivated() {
    const activated = await users.isActivated();

    if (!activated) {
        debug('restartMailIfActivated: skipping restart of mail container since Cloudron is not activated yet');
        return; // not provisioned yet, do not restart container after dns setup
    }

    await util.promisify(restartMail)();
}

async function handleCertChanged() {
    debug('handleCertChanged: will restart if activated');
    await restartMailIfActivated();
}

async function getDomain(domain) {
    assert.strictEqual(typeof domain, 'string');

    const result = await database.query(`SELECT ${MAILDB_FIELDS} FROM mail WHERE domain = ?`, [ domain ]);
    if (result.length === 0) return null;
    return postProcessDomain(result[0]);
}

async function updateDomain(domain, data) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');

    const args = [ ];
    const fields = [ ];
    for (var k in data) {
        if (k === 'catchAll' || k === 'banner') {
            fields.push(`${k}Json = ?`);
            args.push(JSON.stringify(data[k]));
        } else if (k === 'relay') {
            fields.push('relayJson = ?');
            args.push(JSON.stringify(data[k]));
        } else {
            fields.push(k + ' = ?');
            args.push(data[k]);
        }
    }
    args.push(domain);

    const result = await database.query('UPDATE mail SET ' + fields.join(', ') + ' WHERE domain=?', args);
    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Mail domain not found');
}

async function listDomains() {
    const results = await database.query(`SELECT ${MAILDB_FIELDS} FROM mail ORDER BY domain`);
    results.forEach(function (result) { postProcessDomain(result); });
    return results;
}

// https://agari.zendesk.com/hc/en-us/articles/202952749-How-long-can-my-SPF-record-be-
function txtRecordsWithSpf(domain, mailFqdn, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof callback, 'function');

    dns.getDnsRecords('', domain, 'TXT', function (error, txtRecords) {
        if (error) return callback(error);

        debug('txtRecordsWithSpf: current txt records - %j', txtRecords);

        var i, matches, validSpf;

        for (i = 0; i < txtRecords.length; i++) {
            matches = txtRecords[i].match(/^("?v=spf1) /); // DO backend may return without quotes
            if (matches === null) continue;

            // this won't work if the entry is arbitrarily "split" across quoted strings
            validSpf = txtRecords[i].indexOf('a:' + mailFqdn) !== -1;
            break; // there can only be one SPF record
        }

        if (validSpf) return callback(null, null);

        if (!matches) { // no spf record was found, create one
            txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"');
            debug('txtRecordsWithSpf: adding txt record');
        } else { // just add ourself
            txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length);
            debug('txtRecordsWithSpf: inserting txt record');
        }

        return callback(null, txtRecords);
    });
}

function ensureDkimKeySync(mailDomain) {
    assert.strictEqual(typeof mailDomain, 'object');

    const domain = mailDomain.domain;
    const dkimPath = path.join(paths.MAIL_DATA_DIR, `dkim/${domain}`);
    const dkimPrivateKeyFile = path.join(dkimPath, 'private');
    const dkimPublicKeyFile = path.join(dkimPath, 'public');
    const dkimSelectorFile = path.join(dkimPath, 'selector');

    if (safe.fs.existsSync(dkimPublicKeyFile) &&
        safe.fs.existsSync(dkimPublicKeyFile) &&
        safe.fs.existsSync(dkimPublicKeyFile)) {
        debug(`Reusing existing DKIM keys for ${domain}`);
        return null;
    }

    debug(`Generating new DKIM keys for ${domain}`);

    if (!safe.fs.mkdirSync(dkimPath) && safe.error.code !== 'EEXIST') {
        debug('Error creating dkim.', safe.error);
        return new BoxError(BoxError.FS_ERROR, safe.error);
    }

    // https://www.unlocktheinbox.com/dkim-key-length-statistics/ and https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-authentication-dkim-easy.html for key size
    if (!safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024')) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);
    if (!safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM')) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);

    if (!safe.fs.writeFileSync(dkimSelectorFile, mailDomain.dkimSelector, 'utf8')) return new BoxError(BoxError.FS_ERROR, safe.error);

    // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
    if (!safe.fs.chmodSync(dkimPrivateKeyFile, 0o644)) return new BoxError(BoxError.FS_ERROR, safe.error);

    return null;
}

function readDkimPublicKeySync(domain) {
    assert.strictEqual(typeof domain, 'string');

    var dkimPath = path.join(paths.MAIL_DATA_DIR, `dkim/${domain}`);
    var dkimPublicKeyFile = path.join(dkimPath, 'public');

    var publicKey = safe.fs.readFileSync(dkimPublicKeyFile, 'utf8');

    if (publicKey === null) {
        debug('Error reading dkim public key.', safe.error);
        return null;
    }

    // remove header, footer and new lines
    publicKey = publicKey.split('\n').slice(1, -2).join('');

    return publicKey;
}

function upsertDnsRecords(domain, mailFqdn, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof callback, 'function');

    debug(`upsertDnsRecords: updating mail dns records of domain ${domain} and mail fqdn ${mailFqdn}`);

    const getDomainFunc = util.callbackify(getDomain);

    getDomainFunc(domain, function (error, mailDomain) {
        if (error) return callback(error);
        if (!mailDomain) return callback(new BoxError(BoxError.NOT_FOUND, 'mail domain not found'));

        error = ensureDkimKeySync(mailDomain);
        if (error) return callback(error);

        if (process.env.BOX_ENV === 'test') return callback();

        const dkimKey = readDkimPublicKeySync(domain);
        if (!dkimKey) return callback(new BoxError(BoxError.FS_ERROR, 'Failed to read dkim public key'));

        // t=s limits the domainkey to this domain and not it's subdomains
        const dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ `"v=DKIM1; t=s; p=${dkimKey}"` ] };

        let records = [];
        records.push(dkimRecord);
        if (mailDomain.enabled) records.push({ subdomain: '', domain: domain, type: 'MX', values: [ '10 ' + mailFqdn + '.' ] });

        txtRecordsWithSpf(domain, mailFqdn, function (error, txtRecords) {
            if (error) return callback(error);

            if (txtRecords) records.push({ subdomain: '', domain: domain, type: 'TXT', values: txtRecords });

            dns.getDnsRecords('_dmarc', domain, 'TXT', function (error, dmarcRecords) { // only update dmarc if absent. this allows user to set email for reporting
                if (error) return callback(error);

                if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain: domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });

                debug('upsertDnsRecords: will update %j', records);

                async.mapSeries(records, function (record, iteratorCallback) {
                    dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values, iteratorCallback);
                }, function (error, changeIds) {
                    if (error) {
                        debug(`upsertDnsRecords: failed to update: ${error}`);
                        return callback(error);
                    }

                    debug('upsertDnsRecords: records %j added with changeIds %j', records, changeIds);

                    callback(null);
                });
            });
        });
    });
}

function setDnsRecords(domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

    upsertDnsRecords(domain, settings.mailFqdn(), callback);
}

function getLocation(callback) {
    assert.strictEqual(typeof callback, 'function');

    const domain = settings.mailDomain(), fqdn = settings.mailFqdn();
    const subdomain = fqdn.substr(0, fqdn.length - domain.length - 1);

    callback(null, { domain, subdomain });
}

function changeLocation(auditSource, progressCallback, callback) {
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');
    assert.strictEqual(typeof callback, 'function');

    const fqdn = settings.mailFqdn(), domain = settings.mailDomain();
    const subdomain = fqdn.substr(0, fqdn.length - domain.length - 1);

    let progress = 20;
    progressCallback({ percent: progress, message: `Setting up DNS of certs of mail server ${fqdn}` });

    cloudron.setupDnsAndCert(subdomain, domain, auditSource, progressCallback, function (error) {
        if (error) return callback(error);

        domainsList(function (error, allDomains) {
            if (error) return callback(error);

            async.eachOfSeries(allDomains, function (domainObject, idx, iteratorDone) {
                progressCallback({ percent: progress, message: `Updating DNS of ${domainObject.domain}` });
                progress += Math.round(70/allDomains.length);

                upsertDnsRecords(domainObject.domain, fqdn, function (error) { // ignore any errors. we anyway report dns errors in status tab
                    progressCallback({ percent: progress, message: `Updated DNS of ${domainObject.domain}: ${error ? error.message : 'success'}` });
                    iteratorDone();
                });
            }, async function (error) {
                if (error) return callback(error);

                progressCallback({ percent: 90, message: 'Restarting mail server' });

                [error] = await safe(restartMailIfActivated());
                callback(error);
            });
        });
    });
}

async function setLocation(subdomain, domain, auditSource) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const domainObject = await domains.get(domain);
    const fqdn = dns.fqdn(subdomain, domainObject);

    await settings.setMailLocation(domain, fqdn);

    const taskId = await tasks.add(tasks.TASK_CHANGE_MAIL_LOCATION, [ auditSource ]);
    tasks.startTask(taskId, {}, NOOP_CALLBACK);
    await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId });

    return taskId;
}

function onDomainAdded(domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

    if (!settings.mailFqdn()) return callback(); // mail domain is not set yet (when provisioning)

    async.series([
        upsertDnsRecords.bind(null, domain, settings.mailFqdn()), // do this first to ensure DKIM keys
        restartMailIfActivated
    ], callback);
}

function onDomainRemoved(domain, callback) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof callback, 'function');

    restartMail(callback);
}

async function clearDomains() {
    await database.query('DELETE FROM mail', []);
}

// remove all fields that should never be sent out via REST API
function removePrivateFields(domain) {
    let result = _.pick(domain, 'domain', 'enabled', 'mailFromValidation', 'catchAll', 'relay', 'banner');
    if (result.relay.provider !== 'cloudron-smtp') {
        if (result.relay.username === result.relay.password) result.relay.username = constants.SECRET_PLACEHOLDER;
        result.relay.password = constants.SECRET_PLACEHOLDER;
    }
    return result;
}

async function setMailFromValidation(domain, enabled) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof enabled, 'boolean');

    await updateDomain(domain, { mailFromValidation: enabled });

    restartMail(NOOP_CALLBACK); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}

async function setBanner(domain, banner) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof banner, 'object');

    await updateDomain(domain, { banner });

    restartMail(NOOP_CALLBACK);
}

async function setCatchAllAddress(domain, addresses) {
    assert.strictEqual(typeof domain, 'string');
    assert(Array.isArray(addresses));

    await updateDomain(domain, { catchAll: addresses });

    restartMail(NOOP_CALLBACK); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}

async function setMailRelay(domain, relay, options) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof relay, 'object');
    assert.strictEqual(typeof options, 'object');

    const result = await getDomain(domain);
    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'Mail domain not found');

    // inject current username/password
    if (result.relay.provider === relay.provider) {
        if (relay.username === constants.SECRET_PLACEHOLDER) relay.username = result.relay.username;
        if (relay.password === constants.SECRET_PLACEHOLDER) relay.password = result.relay.password;
    }

    if (!options.skipVerify) await verifyRelay(relay);

    await updateDomain(domain, { relay: relay });

    restartMail(NOOP_CALLBACK);
}

async function setMailEnabled(domain, enabled, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof enabled, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');

    await updateDomain(domain, { enabled: enabled });

    restartMail(NOOP_CALLBACK);

    await eventlog.add(enabled ? eventlog.ACTION_MAIL_ENABLED : eventlog.ACTION_MAIL_DISABLED, auditSource, { domain });
}

async function sendTestMail(domain, to) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof to, 'string');

    const result = await getDomain(domain);
    if (!result) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    await mailer.sendTestMail(result.domain);
}

async function listMailboxes(domain, search, page, perPage) {
    assert.strictEqual(typeof domain, 'string');
    assert(typeof search === 'string' || search === null);
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
    const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use

    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
        + ' WHERE m1.domain = ?'
        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
        + searchQuery
        + ' ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ domain, (page-1)*perPage, perPage ]);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function listAllMailboxes(page, perPage) {
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
        + ' ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ (page-1)*perPage, perPage ]);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function getMailboxCount(domain) {
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT COUNT(*) AS total FROM mailboxes WHERE type = ? AND domain = ?', [ exports.TYPE_MAILBOX, domain ]);

    return results[0].total;
}

async function delByDomain(domain) {
    assert.strictEqual(typeof domain, 'string');

    await database.query('DELETE FROM mailboxes WHERE domain = ?', [ domain ]);
}

async function get(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
    if (results.length === 0) return null;

    return postProcessMailbox(results[0]);
}

async function getMailbox(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND type = ? AND domain = ?', [ name, exports.TYPE_MAILBOX, domain ]);
    if (results.length === 0) return null;
    return postProcessMailbox(results[0]);
}

async function addMailbox(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { ownerId, ownerType, active } = data;
    assert.strictEqual(typeof ownerId, 'string');
    assert.strictEqual(typeof ownerType, 'string');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');

    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active) VALUES (?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType, active ]));
    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
    if (error) throw error;

    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
}

async function updateMailbox(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { ownerId, ownerType, active } = data;
    assert.strictEqual(typeof ownerId, 'string');
    assert.strictEqual(typeof ownerType, 'string');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    if (ownerType !== exports.OWNERTYPE_USER && ownerType !== exports.OWNERTYPE_GROUP) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');

    const mailbox = await getMailbox(name, domain);
    if (!mailbox) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');

    const result = await database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ?, active = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, active, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
}

function removeSolrIndex(mailbox, callback) {
    assert.strictEqual(typeof mailbox, 'string');
    assert.strictEqual(typeof callback, 'function');

    services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
        if (error) return callback(error);

        request.post(`https://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`, { timeout: 2000, rejectUnauthorized: false, json: { mailbox } }, function (error, response) {
            if (error) return callback(error);

            if (response.statusCode !== 200) return callback(new Error(`Error removing solr index - ${response.statusCode} ${JSON.stringify(response.body)}`));

            callback(null);
        });
    });
}

async function delMailbox(name, domain, options, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const mailbox =`${name}@${domain}`;
    if (options.deleteMails) {
        const [error] = await safe(shell.promises.sudo('removeMailbox', [ REMOVE_MAILBOX_CMD, mailbox ], {}));
        if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing mailbox: ${error.message}`);
    }

    // deletes aliases as well
    const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    removeSolrIndex(mailbox, NOOP_CALLBACK);
    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
}

async function getAlias(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE name = ? AND type = ? AND domain = ?`, [ name, exports.TYPE_ALIAS, domain ]);
    if (results.length === 0) return null;

    results.forEach(function (result) { postProcessMailbox(result); });

    return results[0];
}

async function getAliases(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const result = await getMailbox(name, domain); // check if mailbox exists
    if (result === null) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');
    return await database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name', [ exports.TYPE_ALIAS, name, domain ]);
}

async function setAliases(name, domain, aliases) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert(Array.isArray(aliases));

    for (let i = 0; i < aliases.length; i++) {
        let name = aliases[i].name.toLowerCase();
        let domain = aliases[i].domain.toLowerCase();

        let error = validateName(name);
        if (error) throw error;

        if (!validator.isEmail(`${name}@${domain}`)) throw new BoxError(BoxError.BAD_FIELD, `Invalid email: ${name}@${domain}`);
        aliases[i] = { name, domain };
    }

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
    if (results.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    let queries = [];
    // clear existing aliases
    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
    aliases.forEach(function (alias) {
        queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
            args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
    });

    const [error] = await safe(database.transaction(queries));
    if (error && error.code === 'ER_DUP_ENTRY' && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
        const aliasMatch = error.message.match(new RegExp(`^ER_DUP_ENTRY: Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'$`));
        if (!aliasMatch) throw new BoxError(BoxError.ALREADY_EXISTS, error.message);

        throw new BoxError(BoxError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`);
    }
    if (error) throw error;
}

async function getLists(domain, search, page, perPage) {
    assert.strictEqual(typeof domain, 'string');
    assert(typeof search === 'string' || search === null);
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ' OR membersJson LIKE ' + mysql.escape('%' + search + '%') + ')';

    query += 'ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ exports.TYPE_LIST, domain, (page-1)*perPage, perPage ]);

    results.forEach(function (result) { postProcessMailbox(result); });

    return results;
}

async function getList(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND name = ? AND domain = ?', [ exports.TYPE_LIST, name, domain ]);
    if (results.length === 0) return null;

    return postProcessMailbox(results[0]);
}

async function addList(name, domain, data, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { members, membersOnly, active } = data;
    assert(Array.isArray(members));
    assert.strictEqual(typeof membersOnly, 'boolean');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    for (let i = 0; i < members.length; i++) {
        if (!validator.isEmail(members[i])) throw new BoxError(BoxError.BAD_FIELD, 'Invalid mail member: ' + members[i]);
    }

    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, membersJson, membersOnly, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_LIST, domain, 'admin', 'user', JSON.stringify(members), membersOnly, active ]));
    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
    if (error) throw error;

    eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
}

async function updateList(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { members, membersOnly, active } = data;
    assert(Array.isArray(members));
    assert.strictEqual(typeof membersOnly, 'boolean');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    for (let i = 0; i < members.length; i++) {
        if (!validator.isEmail(members[i])) throw new BoxError(BoxError.BAD_FIELD, 'Invalid email: ' + members[i]);
    }

    const result = await database.query('UPDATE mailboxes SET membersJson = ?, membersOnly = ?, active = ? WHERE name = ? AND domain = ?',
        [ JSON.stringify(members), membersOnly, active, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
}

async function delList(name, domain, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    // deletes aliases as well
    const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
}

// resolves the members of a list. i.e the lists and aliases
async function resolveList(listName, listDomain) {
    assert.strictEqual(typeof listName, 'string');
    assert.strictEqual(typeof listDomain, 'string');

    const mailDomains = await listDomains();
    const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');

    const list = await getList(listName, listDomain);
    if (!list) throw new BoxError(BoxError.NOT_FOUND, 'List not found');

    let resolvedMembers = [], toResolve = list.members.slice(), visited = []; // slice creates a copy of array

    while (toResolve.length != 0) {
        const toProcess = toResolve.shift();
        const parts = toProcess.split('@');
        const memberName = parts[0].split('+')[0], memberDomain = parts[1];

        if (!mailInDomains.includes(memberDomain)) {  // external domain
            resolvedMembers.push(toProcess);
            continue;
        }

        const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress
        if (visited.includes(member)) {
            debug(`resolveList: list ${listName}@${listDomain} has a recursion at member ${member}`);
            continue;
        }
        visited.push(member);

        const entry = await get(memberName, memberDomain);
        if (!entry) { // let it bounce
            resolvedMembers.push(member);
            continue;
        }

        if (entry.type === exports.TYPE_MAILBOX) { // concrete mailbox
            resolvedMembers.push(member);
        } else if (entry.type === exports.TYPE_ALIAS) { // resolve aliases
            toResolve = toResolve.concat(`${entry.aliasName}@${entry.aliasDomain}`);
        } else { // resolve list members
            toResolve = toResolve.concat(entry.members);
        }
    }

    return { resolvedMembers, list };
}