cloudron-box/src/mail.js

'use strict';

exports = module.exports = {
    getStatus,
    checkConfiguration,

    getLocation,
    setLocation, // triggers the change task
    changeLocation, // does the actual changing

    listDomains,

    getDomain,
    clearDomains,

    generateDkimKey,
    onDomainAdded,
    onDomainRemoved,

    removePrivateFields,

    setDnsRecords,

    validateName,
    validateDisplayName,

    setMailFromValidation,
    setCatchAllAddress,
    setMailRelay,
    setMailEnabled,
    setBanner,

    startMail,
    restartMail,
    handleCertChanged,
    getMailAuth,

    sendTestMail,

    getMailboxCount,
    listMailboxes,
    listAllMailboxes,
    getMailbox,
    addMailbox,
    updateMailbox,
    delMailbox,

    getAlias,
    getAliases,
    setAliases,

    getLists,
    getList,
    addList,
    updateList,
    delList,
    resolveList,

    OWNERTYPE_USER: 'user',
    OWNERTYPE_GROUP: 'group',
    OWNERTYPE_APP: 'app',

    DEFAULT_MEMORY_LIMIT: 512 * 1024 * 1024,

    TYPE_MAILBOX: 'mailbox',
    TYPE_LIST: 'list',
    TYPE_ALIAS: 'alias',

    _delByDomain: delByDomain,
    _updateDomain: updateDomain
};

const assert = require('assert'),
    BoxError = require('./boxerror.js'),
    cloudron = require('./cloudron.js'),
    constants = require('./constants.js'),
    crypto = require('crypto'),
    database = require('./database.js'),
    debug = require('debug')('box:mail'),
    dig = require('./dig.js'),
    dns = require('./dns.js'),
    docker = require('./docker.js'),
    domains = require('./domains.js'),
    eventlog = require('./eventlog.js'),
    hat = require('./hat.js'),
    infra = require('./infra_version.js'),
    mailer = require('./mailer.js'),
    mysql = require('mysql'),
    net = require('net'),
    nodemailer = require('nodemailer'),
    os = require('os'),
    path = require('path'),
    paths = require('./paths.js'),
    reverseProxy = require('./reverseproxy.js'),
    safe = require('safetydance'),
    services = require('./services.js'),
    settings = require('./settings.js'),
    shell = require('./shell.js'),
    smtpTransport = require('nodemailer-smtp-transport'),
    superagent = require('superagent'),
    sysinfo = require('./sysinfo.js'),
    system = require('./system.js'),
    tasks = require('./tasks.js'),
    users = require('./users.js'),
    util = require('util'),
    validator = require('validator'),
    _ = require('underscore');

const DNS_OPTIONS = { timeout: 5000 };
const REMOVE_MAILBOX_CMD = path.join(__dirname, 'scripts/rmmailbox.sh');

const OWNERTYPES = [ exports.OWNERTYPE_USER, exports.OWNERTYPE_GROUP, exports.OWNERTYPE_APP ];

// if you add a field here, listMailboxes has to be updated
const MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain', 'active', 'enablePop3' ].join(',');
const MAILDB_FIELDS = [ 'domain', 'enabled', 'mailFromValidation', 'catchAllJson', 'relayJson', 'dkimKeyJson', 'dkimSelector', 'bannerJson' ].join(',');

function postProcessMailbox(data) {
    data.members = safe.JSON.parse(data.membersJson) || [ ];
    delete data.membersJson;

    data.membersOnly = !!data.membersOnly;
    data.active = !!data.active;
    data.enablePop3 = !!data.enablePop3;

    return data;
}

function postProcessAliases(data) {
    const aliasNames = JSON.parse(data.aliasNames), aliasDomains = JSON.parse(data.aliasDomains);
    delete data.aliasNames;
    delete data.aliasDomains;
    data.aliases = [];
    for (let i = 0; i < aliasNames.length; i++) { // NOTE: aliasNames is [ null ] when no aliases
        if (aliasNames[i]) data.aliases[i] = { name: aliasNames[i], domain: aliasDomains[i] };
    }

    return data;
}

function postProcessDomain(data) {
    data.enabled = !!data.enabled; // int to boolean
    data.mailFromValidation = !!data.mailFromValidation; // int to boolean

    data.catchAll = safe.JSON.parse(data.catchAllJson) || [ ];
    delete data.catchAllJson;

    data.relay = safe.JSON.parse(data.relayJson) || { provider: 'cloudron-smtp' };
    delete data.relayJson;

    data.banner = safe.JSON.parse(data.bannerJson) || { text: null, html: null };
    delete data.bannerJson;

    data.dkimKey = safe.JSON.parse(data.dkimKeyJson) || null;
    delete data.dkimKeyJson;

    return data;
}

function validateName(name) {
    assert.strictEqual(typeof name, 'string');

    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox name must be atleast 1 char');
    if (name.length >= 200) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');

    // also need to consider valid LDAP characters here (e.g '+' is reserved). keep hyphen at the end so it doesn't become a range.
    if (/[^a-zA-Z0-9._-]/.test(name)) return new BoxError(BoxError.BAD_FIELD, 'mailbox name can only contain alphanumerals, dot, hyphen or underscore');

    return null;
}

function validateDisplayName(name) {
    assert.strictEqual(typeof name, 'string');

    if (name.length < 1) return new BoxError(BoxError.BAD_FIELD, 'mailbox display name must be atleast 1 char');
    if (name.length >= 100) return new BoxError(BoxError.BAD_FIELD, 'mailbox name too long');

    return null;
}

async function checkOutboundPort25() {
    const relay = {
        value: 'OK',
        status: false,
        errorMessage: ''
    };

    return await new Promise((resolve) => {
        const client = new net.Socket();
        client.setTimeout(5000);
        client.connect(25, constants.PORT25_CHECK_SERVER);
        client.on('connect', function () {
            relay.status = true;
            relay.value = 'OK';
            client.destroy(); // do not use end() because it still triggers timeout
            resolve(relay);
        });
        client.on('timeout', function () {
            relay.status = false;
            relay.value = `Connect to ${constants.PORT25_CHECK_SERVER} timed out. Check if port 25 (outbound) is blocked`;
            client.destroy();
            relay.errorMessage = `Connect to ${constants.PORT25_CHECK_SERVER} timed out.`;
            resolve(relay);
        });
        client.on('error', function (error) {
            relay.status = false;
            relay.value = `Connect to ${constants.PORT25_CHECK_SERVER} failed: ${error.message}. Check if port 25 (outbound) is blocked`;
            client.destroy();
            relay.errorMessage = `Connect to ${constants.PORT25_CHECK_SERVER} failed: ${error.message}`;
            resolve(relay);
        });
    });
}

async function checkSmtpRelay(relay) {
    const result = {
        value: 'OK',
        status: false,
        errorMessage: ''
    };

    const options = {
        connectionTimeout: 5000,
        greetingTimeout: 5000,
        host: relay.host,
        port: relay.port
    };

    // only set auth if either username or password is provided, some relays auth based on IP (range)
    if (relay.username || relay.password) {
        options.auth = {
            user: relay.username,
            pass: relay.password
        };
    }

    if (relay.acceptSelfSignedCerts) options.tls = { rejectUnauthorized: false };

    const transporter = nodemailer.createTransport(smtpTransport(options));

    const [error] = await safe(util.promisify(transporter.verify)());
    result.status = !error;
    if (error) {
        result.value = result.errorMessage = error.message;
        return result;
    }

    return result;
}

async function verifyRelay(relay) {
    assert.strictEqual(typeof relay, 'object');

    // we used to verify cloudron-smtp with checkOutboundPort25 but that is unreliable given that we just
    // randomly select some smtp server
    if (relay.provider === 'cloudron-smtp' || relay.provider === 'noop') return null;

    const result = await checkSmtpRelay(relay);
    if (result.errorMessage) return new BoxError(BoxError.BAD_FIELD, result.errorMessage);
}

async function checkDkim(mailDomain) {
    assert.strictEqual(typeof mailDomain, 'object');

    const domain = mailDomain.domain;
    const dkim = {
        domain: `${mailDomain.dkimSelector}._domainkey.${domain}`,
        name: `${mailDomain.dkimSelector}._domainkey`,
        type: 'TXT',
        expected: null,
        value: null,
        status: false,
        errorMessage: ''
    };

    const publicKey = mailDomain.dkimKey.publicKey.split('\n').slice(1, -2).join(''); // remove header, footer and new lines

    dkim.expected = `v=DKIM1; t=s; p=${publicKey}`;

    const [error, txtRecords] = await safe(dig.resolve(dkim.domain, dkim.type, DNS_OPTIONS));
    if (error) {
        dkim.errorMessage = error.message;
        return dkim;
    }

    if (txtRecords.length !== 0) {
        dkim.value = txtRecords[0].join('');
        const actual = txtToDict(dkim.value);
        dkim.status = actual.p === publicKey;
    }

    return dkim;
}

async function checkSpf(domain, mailFqdn) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');

    const spf = {
        domain: domain,
        name: '@',
        type: 'TXT',
        value: null,
        expected: 'v=spf1 a:' + mailFqdn + ' ~all',
        status: false,
        errorMessage: ''
    };

    const [error, txtRecords] = await safe(dig.resolve(spf.domain, spf.type, DNS_OPTIONS));
    if (error) {
        spf.errorMessage = error.message;
        return spf;
    }

    let i;
    for (i = 0; i < txtRecords.length; i++) {
        let txtRecord = txtRecords[i].join(''); // https://agari.zendesk.com/hc/en-us/articles/202952749-How-long-can-my-SPF-record-be-
        if (txtRecord.indexOf('v=spf1 ') !== 0) continue; // not SPF
        spf.value = txtRecord;
        spf.status = spf.value.indexOf(' a:' + settings.mailFqdn()) !== -1;
        break;
    }

    if (spf.status) {
        spf.expected = spf.value;
    } else if (i !== txtRecords.length) {
        spf.expected = 'v=spf1 a:' + settings.mailFqdn() + ' ' + spf.value.slice('v=spf1 '.length);
    }

    return spf;
}

async function checkMx(domain, mailFqdn) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');

    const mx = {
        domain: domain,
        name: '@',
        type: 'MX',
        value: null,
        expected: '10 ' + mailFqdn + '.',
        status: false,
        errorMessage: ''
    };

    const [error, mxRecords] = await safe(dig.resolve(mx.domain, mx.type, DNS_OPTIONS));
    if (error) {
        mx.errorMessage = error.message;
        return mx;
    }
    if (mxRecords.length === 0) return mx;

    mx.status = mxRecords.some(mx => mx.exchange === mailFqdn); // this lets use change priority and/or setup backup MX
    mx.value = mxRecords.map(function (r) { return r.priority + ' ' + r.exchange + '.'; }).join(' ');

    if (mx.status) return mx; // MX record is "my."

    // cloudflare might create a conflict subdomain (https://support.cloudflare.com/hc/en-us/articles/360020296512-DNS-Troubleshooting-FAQ)
    const [error2, mxIps] = await safe(dig.resolve(mxRecords[0].exchange, 'A', DNS_OPTIONS));
    if (error2 || mxIps.length !== 1) return mx;

    const [error3, ip] = await safe(sysinfo.getServerIPv4());
    if (error3) return mx;

    mx.status = mxIps[0] === ip;

    return mx;
}

function txtToDict(txt) {
    const dict = {};
    txt.split(';').forEach(function(v) {
        const p = v.trim().split('=');
        dict[p[0]]=p[1];
    });
    return dict;
}

async function checkDmarc(domain) {

    const dmarc = {
        domain: '_dmarc.' + domain,
        name: '_dmarc',
        type: 'TXT',
        value: null,
        expected: 'v=DMARC1; p=reject; pct=100',
        status: false,
        errorMessage: ''
    };

    const [error, txtRecords] = await safe(dig.resolve(dmarc.domain, dmarc.type, DNS_OPTIONS));

    if (error) {
        dmarc.errorMessage = error.message;
        return dmarc;
    }

    if (txtRecords.length !== 0) {
        dmarc.value = txtRecords[0].join('');
        const actual = txtToDict(dmarc.value);
        dmarc.status = actual.v === 'DMARC1'; // see box#666
    }

    return dmarc;
}

async function checkPtr(mailFqdn) {
    assert.strictEqual(typeof mailFqdn, 'string');

    const ptr = {
        domain: null,
        name: null,
        type: 'PTR',
        value: null,
        expected: mailFqdn, // any trailing '.' is added by client software (https://lists.gt.net/spf/devel/7918)
        status: false,
        errorMessage: ''
    };

    const [error, ip] = await safe(sysinfo.getServerIPv4());
    if (error) {
        ptr.errorMessage = error.message;
        return ptr;
    }

    ptr.domain = ip.split('.').reverse().join('.') + '.in-addr.arpa';
    ptr.name = ip;

    const [error2, ptrRecords] = await safe(dig.resolve(ptr.domain, 'PTR', DNS_OPTIONS));
    if (error2) {
        ptr.errorMessage = error2.message;
        return ptr;
    }

    if (ptrRecords.length !== 0) {
        ptr.value = ptrRecords.join(' ');
        ptr.status = ptrRecords.some(function (v) { return v === ptr.expected; });
    }

    return ptr;
}

// https://raw.githubusercontent.com/jawsome/node-dnsbl/master/list.json
const RBL_LIST = [
    {
        'name': 'Abuse.ch',
        'dns': 'spam.abuse.ch',
        'site': 'http://abuse.ch/'
    },

    {
        'name': 'Barracuda',
        'dns': 'b.barracudacentral.org',
        'site': 'http://www.barracudacentral.org/rbl/removal-request'
    },
    {
        'name': 'Composite Blocking List',
        'dns': 'cbl.abuseat.org',
        'site': 'http://www.abuseat.org'
    },
    {
        'name': 'Multi SURBL',
        'dns': 'multi.surbl.org',
        'site': 'http://www.surbl.org'
    },
    {
        'name': 'Passive Spam Block List',
        'dns': 'psbl.surriel.com',
        'site': 'https://psbl.org'
    },
    {
        'name': 'Sorbs Aggregate Zone',
        'dns': 'dnsbl.sorbs.net',
        'site': 'http://dnsbl.sorbs.net/'
    },
    {
        'name': 'Sorbs spam.dnsbl Zone',
        'dns': 'spam.dnsbl.sorbs.net',
        'site': 'http://sorbs.net'
    },
    {
        'name': 'SpamCop',
        'dns': 'bl.spamcop.net',
        'site': 'http://spamcop.net'
    },
    {
        'name': 'SpamHaus Zen',
        'dns': 'zen.spamhaus.org',
        'site': 'http://spamhaus.org'
    },
    {
        'name': 'The Unsubscribe Blacklist(UBL)',
        'dns': 'ubl.unsubscore.com ',
        'site': 'http://www.lashback.com/blacklist/'
    },
    {
        'name': 'UCEPROTECT Network',
        'dns': 'dnsbl-1.uceprotect.net',
        'site': 'http://www.uceprotect.net/en'
    }
];

// this function currently only looks for black lists based on IP. TODO: also look up by domain
async function checkRblStatus(domain) {
    const ip = await sysinfo.getServerIPv4();

    const flippedIp = ip.split('.').reverse().join('.');

    // https://tools.ietf.org/html/rfc5782
    const blacklistedServers = [];
    for (const rblServer of RBL_LIST) {
        const [error, records] = await safe(dig.resolve(flippedIp + '.' + rblServer.dns, 'A', DNS_OPTIONS));
        if (error || !records) continue;    // not listed

        debug(`checkRblStatus: ${domain} (ip: ${flippedIp}) is in the blacklist of ${JSON.stringify(rblServer)}`);

        const result = _.extend({ }, rblServer);

        const [error2, txtRecords] = await safe(dig.resolve(flippedIp + '.' + rblServer.dns, 'TXT', DNS_OPTIONS));
        result.txtRecords = error2 || !txtRecords ? 'No txt record' : txtRecords.map(x => x.join(''));

        debug(`checkRblStatus: ${domain} (error: ${error2.message}) (txtRecords: ${JSON.stringify(txtRecords)})`);

        blacklistedServers.push(result);
    }

    debug(`checkRblStatus: ${domain} (ip: ${ip}) blacklistedServers: ${JSON.stringify(blacklistedServers)})`);

    return { status: blacklistedServers.length === 0, ip, servers: blacklistedServers };
}

async function getStatus(domain) {
    assert.strictEqual(typeof domain, 'string');

    // ensure we always have a valid toplevel properties for the api
    const results = {
        dns: {}, // { mx/dmarc/dkim/spf/ptr: { expected, value, name, domain, type, status } }
        rbl: {}, // { status, ip, servers: [{name,site,dns}]} optional. only for cloudron-smtp
        relay: {} // { status, value } always checked
    };

    const mailFqdn = settings.mailFqdn();

    const mailDomain = await getDomain(domain);
    if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    const checks = [];
    if (mailDomain.enabled) {
        checks.push(
            { what: 'dns.mx', promise: checkMx(domain, mailFqdn) },
            { what: 'dns.dmarc', promise: checkDmarc(domain) }
        );
    }

    if (mailDomain.relay.provider === 'cloudron-smtp') {
        // these tests currently only make sense when using Cloudron's SMTP server at this point
        checks.push(
            { what: 'dns.spf', promise: checkSpf(domain, mailFqdn) },
            { what: 'dns.dkim', promise: checkDkim(mailDomain) },
            { what: 'dns.ptr', promise: checkPtr(mailFqdn) },
            { what: 'relay', promise: checkOutboundPort25() },
            { what: 'rbl', promise: checkRblStatus(domain) },
        );
    } else if (mailDomain.relay.provider !== 'noop') {
        checks.push({ what: 'relay', promise: checkSmtpRelay(mailDomain.relay) });
    }

    // wait for all the checks and record the result
    const responses = await Promise.allSettled(checks.map(c => c.promise));
    for (let i = 0; i < checks.length; i++) {
        const response = responses[i], check = checks[i];
        if (response.status !== 'fulfilled') {
            debug(`check ${check.what} was rejected. This is not expected`);
            continue;
        }

        if (response.value.errorMessage) debug(`Ignored error - ${check.what} : ${response.value.errorMessage}`);
        safe.set(results, checks[i].what, response.value || {});
    }

    return results;
}

async function checkConfiguration() {
    let messages = {};

    const allDomains = await listDomains();

    for (const domainObject of allDomains) {
        const result = await getStatus(domainObject.domain);

        let message = [];

        Object.keys(result.dns).forEach((type) => {
            const record = result.dns[type];
            if (!record.status) message.push(`${type.toUpperCase()} DNS record (${record.type}) did not match.\n    * Hostname: \`${record.name}\`\n    * Expected: \`${record.expected}\`\n    * Actual: \`${record.value}\``);
        });
        if (result.relay && result.relay.status === false) message.push(`Relay error: ${result.relay.value}`);
        if (result.rbl && result.rbl.status === false) { // rbl field contents is optional
            const servers = result.rbl.servers.map((bs) => `[${bs.name}](${bs.site})`); // in markdown
            message.push(`This server's IP \`${result.rbl.ip}\` is blacklisted in the following servers - ${servers.join(', ')}`);
        }

        if (message.length) messages[domainObject.domain] = message;
    }

    // create bulleted list for each domain
    let markdownMessage = '';
    Object.keys(messages).forEach((domain) => {
        markdownMessage += `**${domain}**\n`;
        markdownMessage += messages[domain].map((m) => `* ${m}\n`).join('');
        markdownMessage += '\n\n';
    });

    if (markdownMessage) markdownMessage += 'Email Status is checked every 30 minutes.\n See the [troubleshooting docs](https://docs.cloudron.io/troubleshooting/#mail-dns) for more information.\n';

    return markdownMessage; // empty message means all status checks succeeded
}

async function createMailConfig(mailFqdn) {
    assert.strictEqual(typeof mailFqdn, 'string');

    debug(`createMailConfig: generating mail config with ${mailFqdn}`);

    const mailDomains = await listDomains();

    const mailOutDomains = mailDomains.filter(d => d.relay.provider !== 'noop').map(d => d.domain).join(',');
    const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');

    // mail_domain is used for SRS
    if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/mail.ini`,
        `mail_in_domains=${mailInDomains}\nmail_out_domains=${mailOutDomains}\nmail_server_name=${mailFqdn}\n\n`, 'utf8')) {
        throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
    }

    // enable_outbound makes plugin forward email for relayed mail. non-relayed mail always hits LMTP plugin first
    if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/smtp_forward.ini`, 'enable_outbound=false\ndomain_selector=mail_from\n', 'utf8')) {
        throw new BoxError(BoxError.FS_ERROR, `Could not create smtp forward file: ${safe.error.message}`);
    }

    // create sections for per-domain configuration
    for (const domain of mailDomains) {
        const catchAll = domain.catchAll.map(function (c) { return `${c}@${domain.domain}`; }).join(',');
        const mailFromValidation = domain.mailFromValidation;

        if (!safe.fs.appendFileSync(`${paths.MAIL_CONFIG_DIR}/mail.ini`,
            `[${domain.domain}]\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n\n`, 'utf8')) {
            throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
        }

        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/banner/${domain.domain}.text`, domain.banner.text || '')) throw new BoxError(BoxError.FS_ERROR, `Could not create text banner file: ${safe.error.message}`);
        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/banner/${domain.domain}.html`, domain.banner.html || '')) throw new BoxError(BoxError.FS_ERROR, `Could not create html banner file: ${safe.error.message}`);

        safe.fs.mkdirSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}`, { recursive: true });
        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/public`, domain.dkimKey.publicKey)) throw new BoxError(BoxError.FS_ERROR, `Could not create public key file: ${safe.error.message}`);
        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/private`, domain.dkimKey.privateKey)) throw new BoxError(BoxError.FS_ERROR, `Could not create private key file: ${safe.error.message}`);
        if (!safe.fs.writeFileSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/selector`, domain.dkimSelector)) throw new BoxError(BoxError.FS_ERROR, `Could not create selector file: ${safe.error.message}`);

        // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
        if (!safe.fs.chmodSync(`${paths.MAIL_CONFIG_DIR}/dkim/${domain.domain}/private`, 0o644)) throw new BoxError(BoxError.FS_ERROR, safe.error);

        const relay = domain.relay;

        const enableRelay = relay.provider !== 'cloudron-smtp' && relay.provider !== 'noop',
            host = relay.host || '',
            port = relay.port || 25,
            authType = relay.username ? 'plain' : '',
            username = relay.username || '',
            password = relay.password || '',
            forceFromAddress = relay.forceFromAddress ? 'true' : 'false';

        if (!enableRelay) continue;

        const relayData = `[${domain.domain}]\nenable_outbound=true\nhost=${host}\nport=${port}\nenable_tls=true\nauth_type=${authType}\nauth_user=${username}\nauth_pass=${password}\nforce_from_address=${forceFromAddress}\n\n`;

        if (!safe.fs.appendFileSync(paths.MAIL_CONFIG_DIR + '/smtp_forward.ini', relayData, 'utf8')) {
            throw new BoxError(BoxError.FS_ERROR, `Could not create mail var file: ${safe.error.message}`);
        }
    }

    return mailInDomains.length !== 0 /* allowInbound */;
}

async function configureMail(mailFqdn, mailDomain, serviceConfig) {
    assert.strictEqual(typeof mailFqdn, 'string');
    assert.strictEqual(typeof mailDomain, 'string');
    assert.strictEqual(typeof serviceConfig, 'object');

    // mail (note: 2587 is hardcoded in mail container and app use this port)
    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
    // MAIL_DOMAIN is the domain for which this server is relaying mails
    // mail container uses /app/data for backed up data and /run for restart-able data

    const tag = infra.images.mail.tag;
    const memoryLimit = serviceConfig.memoryLimit || exports.DEFAULT_MEMORY_LIMIT;
    const memory = system.getMemoryAllocation(memoryLimit);
    const cloudronToken = hat(8 * 128), relayToken = hat(8 * 128);

    const bundle = await reverseProxy.getCertificatePath(mailFqdn, mailDomain);

    const dhparamsFilePath = `${paths.MAIL_CONFIG_DIR}/dhparams.pem`;
    const mailCertFilePath = `${paths.MAIL_CONFIG_DIR}/tls_cert.pem`;
    const mailKeyFilePath =  `${paths.MAIL_CONFIG_DIR}/tls_key.pem`;

    if (!safe.child_process.execSync(`cp ${paths.DHPARAMS_FILE} ${dhparamsFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not copy dhparams:' + safe.error.message);
    if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message);
    if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) throw new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message);

    // if the 'yellowtent' user of OS and the 'cloudron' user of mail container don't match, the keys become inaccessible by mail code
    if (!safe.fs.chmodSync(mailKeyFilePath, 0o644)) throw new BoxError(BoxError.FS_ERROR, `Could not chmod key file: ${safe.error.message}`);

    await shell.promises.exec('stopMail', 'docker stop mail || true');
    await shell.promises.exec('removeMail', 'docker rm -f mail || true');

    const allowInbound = await createMailConfig(mailFqdn);

    const ports = allowInbound ? '-p 587:2587 -p 993:9993 -p 4190:4190 -p 25:2587 -p 465:2465 -p 995:9995' : '';
    const readOnly = !serviceConfig.recoveryMode ? '--read-only' : '';
    const cmd = serviceConfig.recoveryMode ? '/bin/bash -c \'echo "Debug mode. Sleeping" && sleep infinity\'' : '';
    const logLevel = serviceConfig.recoveryMode ? 'data' : 'info';

    const runCmd = `docker run --restart=always -d --name="mail" \
                --net cloudron \
                --net-alias mail \
                --log-driver syslog \
                --log-opt syslog-address=udp://127.0.0.1:2514 \
                --log-opt syslog-format=rfc5424 \
                --log-opt tag=mail \
                -m ${memory} \
                --memory-swap ${memoryLimit} \
                --dns 172.18.0.1 \
                --dns-search=. \
                -e CLOUDRON_MAIL_TOKEN="${cloudronToken}" \
                -e CLOUDRON_RELAY_TOKEN="${relayToken}" \
                -e LOGLEVEL=${logLevel} \
                -v "${paths.MAIL_DATA_DIR}:/app/data" \
                -v "${paths.MAIL_CONFIG_DIR}:/etc/mail:ro" \
                ${ports} \
                --label isCloudronManaged=true \
                ${readOnly} -v /run -v /tmp ${tag} ${cmd}`;

    await shell.promises.exec('startMail', runCmd);
}

async function getMailAuth() {
    const data = await docker.inspect('mail');
    const ip = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
    if (!ip) throw new BoxError(BoxError.MAIL_ERROR, 'Error querying mail server IP');

    // extract the relay token for auth
    const env = safe.query(data, 'Config.Env', null);
    if (!env) throw new BoxError(BoxError.MAIL_ERROR, 'Error getting mail env');
    const tmp = env.find(function (e) { return e.indexOf('CLOUDRON_RELAY_TOKEN') === 0; });
    if (!tmp) throw new BoxError(BoxError.MAIL_ERROR, 'Error getting CLOUDRON_RELAY_TOKEN env var');
    const relayToken = tmp.slice('CLOUDRON_RELAY_TOKEN'.length + 1); // +1 for the = sign
    if (!relayToken) throw new BoxError(BoxError.MAIL_ERROR, 'Error parsing CLOUDRON_RELAY_TOKEN');

    return {
        ip,
        port: constants.INTERNAL_SMTP_PORT,
        relayToken
    };
}

async function restartMail() {
    if (process.env.BOX_ENV === 'test' && !process.env.TEST_CREATE_INFRA) return;

    const servicesConfig = await settings.getServicesConfig();
    const mailConfig = servicesConfig['mail'] || {};

    debug(`restartMail: restarting mail container with mailFqdn:${settings.mailFqdn()} mailDomain:${settings.mailDomain()}`);
    await configureMail(settings.mailFqdn(), settings.mailDomain(), mailConfig);
}

async function startMail(existingInfra) {
    assert.strictEqual(typeof existingInfra, 'object');

    await restartMail();
}

async function restartMailIfActivated() {
    const activated = await users.isActivated();

    if (!activated) {
        debug('restartMailIfActivated: skipping restart of mail container since Cloudron is not activated yet');
        return; // not provisioned yet, do not restart container after dns setup
    }

    await restartMail();
}

async function handleCertChanged() {
    debug('handleCertChanged: will restart if activated');
    await restartMailIfActivated();
}

async function getDomain(domain) {
    assert.strictEqual(typeof domain, 'string');

    const result = await database.query(`SELECT ${MAILDB_FIELDS} FROM mail WHERE domain = ?`, [ domain ]);
    if (result.length === 0) return null;
    return postProcessDomain(result[0]);
}

async function updateDomain(domain, data) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');

    const args = [ ];
    const fields = [ ];
    for (const k in data) {
        if (k === 'catchAll' || k === 'banner') {
            fields.push(`${k}Json = ?`);
            args.push(JSON.stringify(data[k]));
        } else if (k === 'relay') {
            fields.push('relayJson = ?');
            args.push(JSON.stringify(data[k]));
        } else {
            fields.push(k + ' = ?');
            args.push(data[k]);
        }
    }
    args.push(domain);

    const result = await database.query('UPDATE mail SET ' + fields.join(', ') + ' WHERE domain=?', args);
    if (result.affectedRows !== 1) throw new BoxError(BoxError.NOT_FOUND, 'Mail domain not found');
}

async function listDomains() {
    const results = await database.query(`SELECT ${MAILDB_FIELDS} FROM mail ORDER BY domain`);
    results.forEach(function (result) { postProcessDomain(result); });
    return results;
}

// https://agari.zendesk.com/hc/en-us/articles/202952749-How-long-can-my-SPF-record-be-
async function txtRecordsWithSpf(domain, mailFqdn) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');

    const txtRecords = await dns.getDnsRecords('', domain, 'TXT');

    debug('txtRecordsWithSpf: current txt records - %j', txtRecords);

    let i, matches, validSpf;

    for (i = 0; i < txtRecords.length; i++) {
        matches = txtRecords[i].match(/^("?v=spf1) /); // DO backend may return without quotes
        if (matches === null) continue;

        // this won't work if the entry is arbitrarily "split" across quoted strings
        validSpf = txtRecords[i].indexOf('a:' + mailFqdn) !== -1;
        break; // there can only be one SPF record
    }

    if (validSpf) return null;

    if (!matches) { // no spf record was found, create one
        txtRecords.push('"v=spf1 a:' + mailFqdn + ' ~all"');
        debug('txtRecordsWithSpf: adding txt record');
    } else { // just add ourself
        txtRecords[i] = matches[1] + ' a:' + mailFqdn + txtRecords[i].slice(matches[1].length);
        debug('txtRecordsWithSpf: inserting txt record');
    }

    return txtRecords;
}

async function generateDkimKey() {
    const publicKeyFilePath = path.join(os.tmpdir(), `dkim-${crypto.randomBytes(4).readUInt32LE(0)}.public`);
    const privateKeyFilePath = path.join(os.tmpdir(), `dkim-${crypto.randomBytes(4).readUInt32LE(0)}.private`);

    // https://www.unlocktheinbox.com/dkim-key-length-statistics/ and https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-authentication-dkim-easy.html for key size
    if (!safe.child_process.execSync(`openssl genrsa -out ${privateKeyFilePath} 1024`)) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);
    if (!safe.child_process.execSync(`openssl rsa -in ${privateKeyFilePath} -out ${publicKeyFilePath} -pubout -outform PEM`)) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);

    const publicKey = safe.fs.readFileSync(publicKeyFilePath, 'utf8');
    if (!publicKey) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    safe.fs.unlinkSync(publicKeyFilePath);

    const privateKey = safe.fs.readFileSync(privateKeyFilePath, 'utf8');
    if (!privateKey) throw new BoxError(BoxError.FS_ERROR, safe.error.message);
    safe.fs.unlinkSync(privateKeyFilePath);

    return { publicKey, privateKey };
}

async function upsertDnsRecords(domain, mailFqdn) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof mailFqdn, 'string');

    debug(`upsertDnsRecords: updating mail dns records of domain ${domain} and mail fqdn ${mailFqdn}`);

    const mailDomain = await getDomain(domain);
    if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    if (process.env.BOX_ENV === 'test') return;

    const publicKey = mailDomain.dkimKey.publicKey.split('\n').slice(1, -2).join(''); // remove header, footer and new lines

    // t=s limits the domainkey to this domain and not it's subdomains
    const dkimRecord = { subdomain: `${mailDomain.dkimSelector}._domainkey`, domain: domain, type: 'TXT', values: [ `"v=DKIM1; t=s; p=${publicKey}"` ] };

    const records = [];
    records.push(dkimRecord);
    if (mailDomain.enabled) records.push({ subdomain: '', domain: domain, type: 'MX', values: [ '10 ' + mailFqdn + '.' ] });

    const txtRecords = await txtRecordsWithSpf(domain, mailFqdn);
    if (txtRecords) records.push({ subdomain: '', domain: domain, type: 'TXT', values: txtRecords });

    const dmarcRecords = await dns.getDnsRecords('_dmarc', domain, 'TXT'); // only update dmarc if absent. this allows user to set email for reporting
    if (dmarcRecords.length === 0) records.push({ subdomain: '_dmarc', domain: domain, type: 'TXT', values: [ '"v=DMARC1; p=reject; pct=100"' ] });

    debug('upsertDnsRecords: will update %j', records);

    for (const record of records) {
        await dns.upsertDnsRecords(record.subdomain, record.domain, record.type, record.values);
    }

    debug('upsertDnsRecords: records %j added', records);
}

async function setDnsRecords(domain) {
    assert.strictEqual(typeof domain, 'string');

    await upsertDnsRecords(domain, settings.mailFqdn());
}

async function getLocation() {
    const domain = settings.mailDomain(), fqdn = settings.mailFqdn();
    const subdomain = fqdn.substr(0, fqdn.length - domain.length - 1);

    return { domain, subdomain };
}

async function changeLocation(auditSource, progressCallback) {
    assert.strictEqual(typeof auditSource, 'object');
    assert.strictEqual(typeof progressCallback, 'function');

    const fqdn = settings.mailFqdn(), domain = settings.mailDomain();
    const subdomain = fqdn.substr(0, fqdn.length - domain.length - 1);

    let progress = 20;
    progressCallback({ percent: progress, message: `Setting up DNS of certs of mail server ${fqdn}` });

    await cloudron.setupDnsAndCert(subdomain, domain, auditSource, (progress) => progressCallback({ message: progress.message })); // remove the percent
    const allDomains = await domains.list();

    for (let idx = 0; idx < allDomains.length; idx++) {
        const domainObject = allDomains[idx];
        progressCallback({ percent: progress, message: `Updating DNS of ${domainObject.domain}` });
        progress += Math.round(70/allDomains.length);

        const [error] = await safe(upsertDnsRecords(domainObject.domain, fqdn)); // ignore any errors. we anyway report dns errors in status tab
        progressCallback({ percent: progress, message: `Updated DNS of ${domainObject.domain}: ${error ? error.message : 'success'}` });
    }

    progressCallback({ percent: 90, message: 'Restarting mail server' });

    await restartMailIfActivated();
}

async function setLocation(subdomain, domain, auditSource) {
    assert.strictEqual(typeof subdomain, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    const domainObject = await domains.get(domain);
    const fqdn = dns.fqdn(subdomain, domainObject);

    await settings.setMailLocation(domain, fqdn);

    const taskId = await tasks.add(tasks.TASK_CHANGE_MAIL_LOCATION, [ auditSource ]);
    tasks.startTask(taskId, {});
    await eventlog.add(eventlog.ACTION_MAIL_LOCATION, auditSource, { subdomain, domain, taskId });

    return taskId;
}

async function onDomainAdded(domain) {
    assert.strictEqual(typeof domain, 'string');

    if (!settings.mailFqdn()) return; // mail domain is not set yet (when provisioning)

    await upsertDnsRecords(domain, settings.mailFqdn());
    await restartMailIfActivated();
}

async function onDomainRemoved(domain) {
    assert.strictEqual(typeof domain, 'string');

    await restartMail();
}

async function clearDomains() {
    await database.query('DELETE FROM mail', []);
}

// remove all fields that should never be sent out via REST API
function removePrivateFields(domain) {
    let result = _.pick(domain, 'domain', 'enabled', 'mailFromValidation', 'catchAll', 'relay', 'banner');
    if (result.relay.provider !== 'cloudron-smtp') {
        if (result.relay.username === result.relay.password) result.relay.username = constants.SECRET_PLACEHOLDER;
        result.relay.password = constants.SECRET_PLACEHOLDER;
    }
    return result;
}

async function setMailFromValidation(domain, enabled) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof enabled, 'boolean');

    await updateDomain(domain, { mailFromValidation: enabled });

    safe(restartMail(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}

async function setBanner(domain, banner) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof banner, 'object');

    await updateDomain(domain, { banner });

    safe(restartMail(), { debug });
}

async function setCatchAllAddress(domain, addresses) {
    assert.strictEqual(typeof domain, 'string');
    assert(Array.isArray(addresses));

    await updateDomain(domain, { catchAll: addresses });

    safe(restartMail(), { debug }); // have to restart mail container since haraka cannot watch symlinked config files (mail.ini)
}

async function setMailRelay(domain, relay, options) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof relay, 'object');
    assert.strictEqual(typeof options, 'object');

    const result = await getDomain(domain);
    if (!domain) throw new BoxError(BoxError.NOT_FOUND, 'Mail domain not found');

    // inject current username/password
    if (result.relay.provider === relay.provider) {
        if (relay.username === constants.SECRET_PLACEHOLDER) relay.username = result.relay.username;
        if (relay.password === constants.SECRET_PLACEHOLDER) relay.password = result.relay.password;
    }

    if (!options.skipVerify) {
        const error = await verifyRelay(relay);
        if (error) throw error;
    }

    await updateDomain(domain, { relay });

    safe(restartMail(), { debug });
}

async function setMailEnabled(domain, enabled, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof enabled, 'boolean');
    assert.strictEqual(typeof auditSource, 'object');

    await updateDomain(domain, { enabled: enabled });

    safe(restartMail(), { debug });

    await eventlog.add(enabled ? eventlog.ACTION_MAIL_ENABLED : eventlog.ACTION_MAIL_DISABLED, auditSource, { domain });
}

async function sendTestMail(domain, to) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof to, 'string');

    const result = await getDomain(domain);
    if (!result) throw new BoxError(BoxError.NOT_FOUND, 'mail domain not found');

    await mailer.sendTestMail(result.domain, to);
}

async function listMailboxes(domain, search, page, perPage) {
    assert.strictEqual(typeof domain, 'string');
    assert(typeof search === 'string' || search === null);
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    const escapedSearch = mysql.escape('%' + search + '%'); // this also quotes the string
    const searchQuery = search ? ` HAVING (name LIKE ${escapedSearch} OR aliasNames LIKE ${escapedSearch} OR aliasDomains LIKE ${escapedSearch})` : ''; // having instead of where because of aggregated columns use

    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3 '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
        + ' WHERE m1.domain = ?'
        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
        + searchQuery
        + ' ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ domain, (page-1)*perPage, perPage ]);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function listAllMailboxes(page, perPage) {
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    const query = 'SELECT m1.name AS name, m1.domain AS domain, m1.ownerId AS ownerId, m1.ownerType as ownerType, m1.active as active, JSON_ARRAYAGG(m2.name) AS aliasNames, JSON_ARRAYAGG(m2.domain) AS aliasDomains, m1.enablePop3 AS enablePop3 '
        + ` FROM (SELECT * FROM mailboxes WHERE type='${exports.TYPE_MAILBOX}') AS m1`
        + ` LEFT JOIN (SELECT * FROM mailboxes WHERE type='${exports.TYPE_ALIAS}') AS m2`
        + '     ON m1.name=m2.aliasName AND m1.domain=m2.aliasDomain AND m1.ownerId=m2.ownerId'
        + ' GROUP BY m1.name, m1.domain, m1.ownerId'
        + ' ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ (page-1)*perPage, perPage ]);

    results.forEach(postProcessMailbox);
    results.forEach(postProcessAliases);

    return results;
}

async function getMailboxCount(domain) {
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT COUNT(*) AS total FROM mailboxes WHERE type = ? AND domain = ?', [ exports.TYPE_MAILBOX, domain ]);

    return results[0].total;
}

async function delByDomain(domain) {
    assert.strictEqual(typeof domain, 'string');

    await database.query('DELETE FROM mailboxes WHERE domain = ?', [ domain ]);
}

async function get(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
    if (results.length === 0) return null;

    return postProcessMailbox(results[0]);
}

async function getMailbox(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND type = ? AND domain = ?', [ name, exports.TYPE_MAILBOX, domain ]);
    if (results.length === 0) return null;
    return postProcessMailbox(results[0]);
}

async function addMailbox(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { ownerId, ownerType, active } = data;
    assert.strictEqual(typeof ownerId, 'string');
    assert.strictEqual(typeof ownerType, 'string');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    if (!OWNERTYPES.includes(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');

    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, active) VALUES (?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType, active ]));
    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
    if (error && error.code === 'ER_NO_REFERENCED_ROW_2' && error.sqlMessage.includes('mailboxes_domain_constraint')) throw new BoxError(BoxError.NOT_FOUND, `no such domain '${domain}'`);
    if (error) throw error;

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_ADD, auditSource, { name, domain, ownerId, ownerType, active });
}

async function updateMailbox(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { ownerId, ownerType, active, enablePop3 } = data;
    assert.strictEqual(typeof ownerId, 'string');
    assert.strictEqual(typeof ownerType, 'string');
    assert.strictEqual(typeof active, 'boolean');
    assert.strictEqual(typeof enablePop3, 'boolean');

    name = name.toLowerCase();

    if (!OWNERTYPES.includes(ownerType)) throw new BoxError(BoxError.BAD_FIELD, 'bad owner type');

    const mailbox = await getMailbox(name, domain);
    if (!mailbox) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');

    const result = await database.query('UPDATE mailboxes SET ownerId = ?, ownerType = ?, active = ?, enablePop3 = ? WHERE name = ? AND domain = ?', [ ownerId, ownerType, active, enablePop3, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldUserId: mailbox.userId, ownerId, ownerType, active });
}

async function removeSolrIndex(mailbox) {
    assert.strictEqual(typeof mailbox, 'string');

    const addonDetails = await services.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN');

    const [error, response] = await safe(superagent.post(`http://${addonDetails.ip}:3000/solr_delete_index?access_token=${addonDetails.token}`)
        .timeout(2000)
        .send({ mailbox })
        .ok(() => true));

    if (error) throw new BoxError(BoxError.MAIL_ERROR, `Could not remove solr index: ${error.message}`);

    if (response.status !== 200) throw new BoxError(BoxError.MAIL_ERROR, `Error removing solr index - ${response.status} ${JSON.stringify(response.body)}`);
}

async function delMailbox(name, domain, options, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof options, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const mailbox =`${name}@${domain}`;
    if (options.deleteMails) {
        const [error] = await safe(shell.promises.sudo('removeMailbox', [ REMOVE_MAILBOX_CMD, mailbox ], {}));
        if (error) throw new BoxError(BoxError.FS_ERROR, `Error removing mailbox: ${error.message}`);
    }

    // deletes aliases as well
    const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    const [error] = await safe(removeSolrIndex(mailbox));
    if (error) debug(`delMailbox: failed to remove solr index: ${error.message}`);

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
}

async function getAlias(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE name = ? AND type = ? AND domain = ?`, [ name, exports.TYPE_ALIAS, domain ]);
    if (results.length === 0) return null;

    results.forEach(function (result) { postProcessMailbox(result); });

    return results[0];
}

async function getAliases(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const result = await getMailbox(name, domain); // check if mailbox exists
    if (result === null) throw new BoxError(BoxError.NOT_FOUND, 'No such mailbox');
    return await database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name', [ exports.TYPE_ALIAS, name, domain ]);
}

async function setAliases(name, domain, aliases, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert(Array.isArray(aliases));
    assert.strictEqual(typeof auditSource, 'object');

    for (let i = 0; i < aliases.length; i++) {
        const name = aliases[i].name.toLowerCase();
        const domain = aliases[i].domain.toLowerCase();

        const error = validateName(name);
        if (error) throw error;

        const mailDomain = await getDomain(domain);
        if (!mailDomain) throw new BoxError(BoxError.NOT_FOUND, `mail domain ${domain} not found`);

        aliases[i] = { name, domain };
    }

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ?', [ name, domain ]);
    if (results.length === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    const queries = [];
    // clear existing aliases
    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
    for (const alias of aliases) {
        queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?, ?)',
            args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId, results[0].ownerType ] });
    }

    const [error] = await safe(database.transaction(queries));
    if (error && error.code === 'ER_DUP_ENTRY' && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
        const aliasMatch = error.message.match(new RegExp(`^ER_DUP_ENTRY: Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'$`));
        if (!aliasMatch) throw new BoxError(BoxError.ALREADY_EXISTS, error.message);

        throw new BoxError(BoxError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`);
    }
    if (error) throw error;

    await eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, aliases });
}

async function getLists(domain, search, page, perPage) {
    assert.strictEqual(typeof domain, 'string');
    assert(typeof search === 'string' || search === null);
    assert.strictEqual(typeof page, 'number');
    assert.strictEqual(typeof perPage, 'number');

    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ' OR membersJson LIKE ' + mysql.escape('%' + search + '%') + ')';

    query += 'ORDER BY name LIMIT ?,?';

    const results = await database.query(query, [ exports.TYPE_LIST, domain, (page-1)*perPage, perPage ]);

    results.forEach(function (result) { postProcessMailbox(result); });

    return results;
}

async function getList(name, domain) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');

    const results = await database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND name = ? AND domain = ?', [ exports.TYPE_LIST, name, domain ]);
    if (results.length === 0) return null;

    return postProcessMailbox(results[0]);
}

async function addList(name, domain, data, auditSource) {
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { members, membersOnly, active } = data;
    assert(Array.isArray(members));
    assert.strictEqual(typeof membersOnly, 'boolean');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    for (let i = 0; i < members.length; i++) {
        if (!validator.isEmail(members[i])) throw new BoxError(BoxError.BAD_FIELD, 'Invalid mail member: ' + members[i]);
    }

    [error] = await safe(database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, membersJson, membersOnly, active) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', [ name, exports.TYPE_LIST, domain, 'admin', 'user', JSON.stringify(members), membersOnly, active ]));
    if (error && error.code === 'ER_DUP_ENTRY') throw new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists');
    if (error) throw error;

    await eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly, active });
}

async function updateList(name, domain, data, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof data, 'object');
    assert.strictEqual(typeof auditSource, 'object');

    const { members, membersOnly, active } = data;
    assert(Array.isArray(members));
    assert.strictEqual(typeof membersOnly, 'boolean');
    assert.strictEqual(typeof active, 'boolean');

    name = name.toLowerCase();

    let error = validateName(name);
    if (error) throw error;

    for (let i = 0; i < members.length; i++) {
        if (!validator.isEmail(members[i])) throw new BoxError(BoxError.BAD_FIELD, 'Invalid email: ' + members[i]);
    }

    const result = await database.query('UPDATE mailboxes SET membersJson = ?, membersOnly = ?, active = ? WHERE name = ? AND domain = ?',
        [ JSON.stringify(members), membersOnly, active, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    await eventlog.add(eventlog.ACTION_MAIL_LIST_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly, active });
}

async function delList(name, domain, auditSource) {
    assert.strictEqual(typeof name, 'string');
    assert.strictEqual(typeof domain, 'string');
    assert.strictEqual(typeof auditSource, 'object');

    // deletes aliases as well
    const result = await database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ]);
    if (result.affectedRows === 0) throw new BoxError(BoxError.NOT_FOUND, 'Mailbox not found');

    await eventlog.add(eventlog.ACTION_MAIL_LIST_REMOVE, auditSource, { name, domain });
}

// resolves the members of a list. i.e the lists and aliases
async function resolveList(listName, listDomain) {
    assert.strictEqual(typeof listName, 'string');
    assert.strictEqual(typeof listDomain, 'string');

    const mailDomains = await listDomains();
    const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');

    const list = await getList(listName, listDomain);
    if (!list) throw new BoxError(BoxError.NOT_FOUND, 'List not found');

    let resolvedMembers = [], toResolve = list.members.slice(), visited = []; // slice creates a copy of array

    while (toResolve.length != 0) {
        const toProcess = toResolve.shift();
        const parts = toProcess.split('@');
        const memberName = parts[0].split('+')[0], memberDomain = parts[1];

        if (!mailInDomains.includes(memberDomain)) {  // external domain
            resolvedMembers.push(toProcess);
            continue;
        }

        const member =`${memberName}@${memberDomain}`; // cleaned up without any '+' subaddress
        if (visited.includes(member)) {
            debug(`resolveList: list ${listName}@${listDomain} has a recursion at member ${member}`);
            continue;
        }
        visited.push(member);

        const entry = await get(memberName, memberDomain);
        if (!entry) { // let it bounce
            resolvedMembers.push(member);
            continue;
        }

        if (entry.type === exports.TYPE_MAILBOX) { // concrete mailbox
            resolvedMembers.push(member);
        } else if (entry.type === exports.TYPE_ALIAS) { // resolve aliases
            toResolve = toResolve.concat(`${entry.aliasName}@${entry.aliasDomain}`);
        } else { // resolve list members
            toResolve = toResolve.concat(entry.members);
        }
    }

    return { resolvedMembers, list };
}