Add 2.1.1 changes

17.12.0-ce has strange issues like https://github.com/moby/moby/issues/34097

.gitattributes

@@ -1,6 +1,7 @@
 # following files are skipped when exporting using git archive
 test export-ignore
-docs export-ignore
+.jshintrc export-ignore
+.gitlab export-ignore
 .gitattributes export-ignore
 .gitignore export-ignore
 

.gitlab/issue_templates/Bug.md

@@ -0,0 +1,6 @@
+Please do not use this issue tracker for support requests and bug reports.
+This issue tracker is used by the Cloudron development team to track actual
+bugs in the code.
+
+Please use the forum at https://forum.cloudron.io to report bugs. For
+confidential issues, please email us at support@cloudron.io.

.gitlab/issue_templates/Feature.md

@@ -0,0 +1,7 @@
+Please do not use this issue tracker for support requests and feature reports.
+This issue tracker is used by the Cloudron development team to track issues in
+the code.
+
+Please use the forum at https://forum.cloudron.io to report bugs. For
+confidential issues, please email us at support@cloudron.io.
+

CHANGES

@@ -1223,3 +1223,38 @@
 * scheduler: do not start cron jobs all at once
 * scheduler: give cron jobs a grace period of 30 minutes to complete
 
+[2.0.1]
+* Multi-domain support
+* Update Haraka to 2.8.18
+* Split box and app autoupdate pattern settings
+* Stop and disable any pre-installed postfix server
+* Migrate altDomain as a manual DNS provider
+* Use node's native dns resolve instead of dig
+* DNS records can now be a A record or a CNAME record
+* Fix generation of fallback certificates to include naked domain
+* Merge multi-string DKIM records
+* scheduler: do not start cron jobs all at once
+* scheduler: give cron jobs a grace period of 30 minutes to complete
+* Rework the eventlog view
+* App clone now clones the robotsTxt and backup settings
+
+[2.1.0]
+* Make S3 backend work reliably with slow internet connections
+* Update docker to 18.03.0-ce
+* Finalize the Email and Mailbox API
+* Move mailbox settings from users to email view
+* mail: fix issue where hosts with valid SPF for a Cloudron domain are unable to send mail to Cloudron
+* mail: fix crash when bounce emails have a null sender
+* Add CSP header for dashboard
+* Add support for installing private docker images
+
+[2.1.1]
+* Make S3 backend work reliably with slow internet connections
+* Update docker to 18.03.0-ce
+* Finalize the Email and Mailbox API
+* Move mailbox settings from users to email view
+* mail: fix issue where hosts with valid SPF for a Cloudron domain are unable to send mail to Cloudron
+* mail: fix crash when bounce emails have a null sender
+* Add CSP header for dashboard
+* Add support for installing private docker images
+

README.md

@@ -48,6 +48,11 @@ apps up-to-date and secure.
 * [Selfhosting](https://cloudron.io/documentation/installation/) - [Pricing](https://cloudron.io/pricing.html)
 * [Managed Hosting](https://cloudron.io/managed.html)
 
+**Note:** This repo is a small part of what gets installed on your server - there is
+the dashboard, database addons, graph container, base image etc. Cloudron also relies
+on external services such as the App Store for apps to be installed. As such, don't
+clone this repo and npm install and expect something to work.
+
 ## Documentation
 
 * [Documentation](https://cloudron.io/documentation/)

migrations/20180405222626-mailboxes-add-members.js

@@ -0,0 +1,51 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    var users = { }, groupMembers = { };
+
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN membersJson TEXT'),
+        function getUsers(done) {
+            db.all('SELECT * from users', [ ], function (error, results) {
+                if (error) return done(error);
+
+                results.forEach(function (result) { users[result.id] = result; });
+
+                done();
+            });
+        },
+        function getGroups(done) {
+            db.all('SELECT id, name, GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                ' GROUP BY groups.id', [ ], function (error, results) {
+                if (error) return done(error);
+
+                results.forEach(function (result) {
+                    var userIds = result.userIds ? result.userIds.split(',') : [];
+                    var members = userIds.map(function (id) { return users[id].username; });
+                    groupMembers[result.id] = members;
+                });
+
+                done();
+            });
+        },
+        function removeGroupIdAndSetMembers(done) {
+            async.eachSeries(Object.keys(groupMembers), function (gid, iteratorDone) {
+                console.log(`Migrating group id ${gid} to ${JSON.stringify(groupMembers[gid])}`);
+
+                db.runSql('UPDATE mailboxes SET membersJson = ?, ownerId = ? WHERE ownerId = ?', [ JSON.stringify(groupMembers[gid]), 'admin', gid ], iteratorDone);
+            }, done);
+        },
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN membersJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20180408014539-mailboxes-add-type.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var async = require('async');
+
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN type VARCHAR(16)'),
+        function addMailboxType(done) {
+            db.all('SELECT * from mailboxes', [ ], function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (mailbox, iteratorCallback) {
+                    let type = 'mailbox';
+                    if (mailbox.aliasTarget) {
+                        type = 'alias';
+                    } else if (mailbox.membersJson) {
+                        type = 'list';
+                    }
+                    db.runSql('UPDATE mailboxes SET type = ? WHERE name = ?', [ type, mailbox.name ], iteratorCallback);
+                }, done);
+            });
+        },
+        db.runSql.bind(db, 'ALTER TABLE mailboxes MODIFY type VARCHAR(16) NOT NULL'),
+        db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE mailboxes DROP COLUMN membersJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -129,7 +129,7 @@ CREATE TABLE IF NOT EXISTS eventlog(
     action VARCHAR(128) NOT NULL,
     source TEXT, /* { userId, username, ip }. userId can be null for cron,sysadmin */
     data TEXT, /* free flowing json based on action */
-    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+    createdAt TIMESTAMP(2) NOT NULL,
 
     PRIMARY KEY (id));
 
@@ -164,9 +164,11 @@ CREATE TABLE IF NOT EXISTS mail(
 */
 CREATE TABLE IF NOT EXISTS mailboxes(
     name VARCHAR(128) NOT NULL,
+    type VARCHAR(16) NOT NULL, /* 'mailbox', 'alias', 'list' */
     ownerId VARCHAR(128) NOT NULL, /* app id or user id or group id */
     ownerType VARCHAR(16) NOT NULL, /* 'app' or 'user' or 'group' */
     aliasTarget VARCHAR(128), /* the target name type is an alias */
+    membersJson TEXT, /* members of a group */
     creationTime TIMESTAMP,
     domain VARCHAR(128),
 

package.json

@@ -77,17 +77,7 @@
     "ws": "^3.3.3"
   },
   "devDependencies": {
-    "bootstrap-sass": "^3.3.3",
     "expect.js": "*",
-    "gulp": "^3.9.1",
-    "gulp-autoprefixer": "^4.0.0",
-    "gulp-concat": "^2.4.3",
-    "gulp-cssnano": "^2.1.0",
-    "gulp-ejs": "^3.1.0",
-    "gulp-sass": "^3.1.0",
-    "gulp-serve": "^1.0.0",
-    "gulp-sourcemaps": "^2.6.1",
-    "gulp-uglify": "^3.0.0",
     "hock": "^1.3.2",
     "istanbul": "*",
     "js2xmlparser": "^3.0.0",
@@ -96,7 +86,7 @@
     "nock": "^9.0.14",
     "node-sass": "^4.6.1",
     "readdirp": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz",
-    "yargs": "^10.1.2"
+    "rimraf": "^2.6.2"
   },
   "scripts": {
     "migrate_local": "DATABASE_URL=mysql://root:@localhost/box node_modules/.bin/db-migrate up",
@@ -106,6 +96,6 @@
     "postmerge": "/bin/true",
     "precommit": "/bin/true",
     "prepush": "npm test",
-    "webadmin": "node_modules/.bin/gulp"
+    "dashboard": "node_modules/.bin/gulp"
   }
 }

scripts/cloudron-setup

@@ -2,16 +2,6 @@
 
 set -eu -o pipefail
 
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $(lsb_release -rs) != "16.04" ]]; then
-    echo "Cloudron requires Ubuntu 16.04" > /dev/stderr
-    exit 1
-fi
-
 # change this to a hash when we make a upgrade release
 readonly LOG_FILE="/var/log/cloudron-setup.log"
 readonly DATA_FILE="/root/cloudron-install-data.json"
@@ -85,6 +75,18 @@ while true; do
     esac
 done
 
+# Only --help works as non-root
+if [[ ${EUID} -ne 0 ]]; then
+    echo "This script should be run as root." > /dev/stderr
+    exit 1
+fi
+
+# Only --help works with mismatched ubuntu
+if [[ $(lsb_release -rs) != "16.04" ]]; then
+    echo "Cloudron requires Ubuntu 16.04" > /dev/stderr
+    exit 1
+fi
+
 # validate arguments in the absence of data
 if [[ -z "${provider}" ]]; then
     echo "--provider is required (azure, cloudscale, digitalocean, ec2, exoscale, hetzner, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic)"
@@ -203,10 +205,10 @@ while true; do
     sleep 10
 done
 
-echo -e "\n\n${GREEN}Visit https://<IP> to finish setup once the server has rebooted.${DONE}"
+echo -e "\n\n${GREEN}Visit https://<IP> and accept the self-signed certificate to finish setup.${DONE}"
 
 if [[ "${rebootServer}" == "true" ]]; then
-    echo -e "\nRebooting this server now to let changes take effect.\n"
+    echo -e "\n${RED}Rebooting this server now to let changes take effect.${DONE}\n"
     systemctl stop mysql # sometimes mysql ends up having corrupt privilege tables
     systemctl reboot
 fi

scripts/createReleaseTarball

@@ -29,8 +29,8 @@ if ! $(cd "${SOURCE_DIR}" && git diff --exit-code >/dev/null); then
     exit 1
 fi
 
-if ! $(cd "${SOURCE_DIR}/../webadmin" && git diff --exit-code >/dev/null); then
-    echo "You have local changes in webadmin, stash or commit them to proceed"
+if ! $(cd "${SOURCE_DIR}/../dashboard" && git diff --exit-code >/dev/null); then
+    echo "You have local changes in dashboard, stash or commit them to proceed"
     exit 1
 fi
 
@@ -42,56 +42,39 @@ fi
 box_version=$(cd "${SOURCE_DIR}" && git rev-parse "HEAD")
 branch=$(git rev-parse --abbrev-ref HEAD)
 if [[ "${branch}" == "master" ]]; then
-    webadmin_version=$(cd "${SOURCE_DIR}/../webadmin" && git rev-parse "${branch}")
+    dashboard_version=$(cd "${SOURCE_DIR}/../dashboard" && git rev-parse "${branch}")
 else
-    webadmin_version=$(cd "${SOURCE_DIR}/../webadmin" && git fetch && git rev-parse "origin/${branch}")
+    dashboard_version=$(cd "${SOURCE_DIR}/../dashboard" && git fetch && git rev-parse "origin/${branch}")
 fi
 bundle_dir=$(mktemp -d -t box 2>/dev/null || mktemp -d box-XXXXXXXXXX --tmpdir=$TMPDIR)
-[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${box_version:0:10}-${webadmin_version:0:10}.tar.gz"
+[[ -z "$bundle_file" ]] && bundle_file="${TMPDIR}/box-${box_version:0:10}-${dashboard_version:0:10}.tar.gz"
 
 chmod "o+rx,g+rx" "${bundle_dir}" # otherwise extracted tarball director won't be readable by others/group
-echo "Checking out code box version [${box_version}] and webadmin version [${webadmin_version}] into ${bundle_dir}"
+echo "==> Checking out code box version [${box_version}] and dashboard version [${dashboard_version}] into ${bundle_dir}"
 (cd "${SOURCE_DIR}" && git archive --format=tar ${box_version} | (cd "${bundle_dir}" && tar xf -))
-(cd "${SOURCE_DIR}/../webadmin" && git archive --format=tar ${webadmin_version} | (cd "${bundle_dir}" && tar xf -))
-(cp "${SOURCE_DIR}/../webadmin/LICENSE" "${bundle_dir}")
+(cd "${SOURCE_DIR}/../dashboard" && git archive --format=tar ${dashboard_version} | (mkdir -p "${bundle_dir}/dashboard.build" && cd "${bundle_dir}/dashboard.build" && tar xf -))
+(cp "${SOURCE_DIR}/../dashboard/LICENSE" "${bundle_dir}")
 
-if diff "${TMPDIR}/boxtarball.cache/package-lock.json.all" "${bundle_dir}/package-lock.json" >/dev/null 2>&1; then
-    echo "Reusing dev modules from cache"
-    cp -r "${TMPDIR}/boxtarball.cache/node_modules-all/." "${bundle_dir}/node_modules"
-else
-    echo "Installing modules with dev dependencies"
-    (cd "${bundle_dir}" && npm install)
+echo "==> Installing modules for dashboard asset generation"
+(cd "${bundle_dir}/dashboard.build" && npm install --production)
 
-    echo "Caching dev dependencies"
-    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-all"
-    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-all/"
-    cp "${bundle_dir}/package-lock.json" "${TMPDIR}/boxtarball.cache/package-lock.json.all"
-fi
+echo "==> Building dashboard assets"
+(cd "${bundle_dir}/dashboard.build" && ./node_modules/.bin/gulp --revision ${dashboard_version})
 
-echo "Building webadmin assets"
-(cd "${bundle_dir}" && ./node_modules/.bin/gulp)
+echo "==> Move built dashboard assets into destination"
+mkdir -p "${bundle_dir}/dashboard"
+mv "${bundle_dir}/dashboard.build/dist" "${bundle_dir}/dashboard/"
 
-echo "Remove intermediate files required at build-time only"
-rm -rf "${bundle_dir}/node_modules/"
-rm -rf "${bundle_dir}/webadmin/src"
-rm -rf "${bundle_dir}/gulpfile.js"
+echo "==> Cleanup dashboard build artifacts"
+rm -rf "${bundle_dir}/dashboard.build"
 
-if diff "${TMPDIR}/boxtarball.cache/package-lock.json.prod" "${bundle_dir}/package-lock.json" >/dev/null 2>&1; then
-    echo "Reusing prod modules from cache"
-    cp -r "${TMPDIR}/boxtarball.cache/node_modules-prod/." "${bundle_dir}/node_modules"
-else
-    echo "Installing modules for production"
-    (cd "${bundle_dir}" && npm install --production --no-optional)
+echo "==> Installing toplevel node modules"
+(cd "${bundle_dir}" && npm install --production --no-optional)
 
-    echo "Caching prod dependencies"
-    mkdir -p "${TMPDIR}/boxtarball.cache/node_modules-prod"
-    rsync -a --delete "${bundle_dir}/node_modules/" "${TMPDIR}/boxtarball.cache/node_modules-prod/"
-    cp "${bundle_dir}/package-lock.json" "${TMPDIR}/boxtarball.cache/package-lock.json.prod"
-fi
-
-echo "Create final tarball"
+echo "==> Create final tarball"
 (cd "${bundle_dir}" && tar czf "${bundle_file}" .)
-echo "Cleaning up ${bundle_dir}"
+
+echo "==> Cleaning up ${bundle_dir}"
 rm -rf "${bundle_dir}"
 
-echo "Tarball saved at ${bundle_file}"
+echo "==> Tarball saved at ${bundle_file}"

scripts/installer.sh

@@ -35,11 +35,11 @@ while true; do
 done
 
 echo "==> installer: updating docker"
-if [[ $(docker version --format {{.Client.Version}}) != "17.09.0-ce" ]]; then
-    $curl -sL https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_17.09.0~ce-0~ubuntu_amd64.deb -o /tmp/docker.deb
+if [[ $(docker version --format {{.Client.Version}}) != "18.03.0-ce" ]]; then
+    $curl -sL https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_18.03.0~ce-0~ubuntu_amd64.deb -o /tmp/docker.deb
 
     # https://download.docker.com/linux/ubuntu/dists/xenial/stable/binary-amd64/Packages
-    if [[ $(sha256sum /tmp/docker.deb | cut -d' ' -f1) != "d33f6eb134f0ab0876148bd96de95ea47d583d7f2cddfdc6757979453f9bd9bf" ]]; then
+    if [[ $(sha256sum /tmp/docker.deb | cut -d' ' -f1) != "1f7315b5723b849fe542fe973b0edb4164a0200e926d386ac14363a968f9e4fc" ]]; then
         echo "==> installer: docker binary download is corrupt"
         exit 5
     fi

setup/start.sh

@@ -214,8 +214,8 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 }
 CONF_END
 
-echo "==> Creating config.json for webadmin"
-cat > "${BOX_SRC_DIR}/webadmin/dist/config.json" <<CONF_END
+echo "==> Creating config.json for dashboard"
+cat > "${BOX_SRC_DIR}/dashboard/dist/config.json" <<CONF_END
 {
     "webServerOrigin": "${arg_web_server_origin}"
 }

setup/start/nginx/appconfig.ejs

@@ -66,8 +66,9 @@ server {
     # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
     ssl_prefer_server_ciphers on;
     ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
-    # ciphers according to https://weakdh.org/sysadmin.html
-    ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
+
+    # ciphers according to https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.10.3&openssl=1.0.2g&hsts=yes&profile=modern
+    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
     ssl_dhparam /home/yellowtent/boxdata/dhparams.pem;
     add_header Strict-Transport-Security "max-age=15768000";
 
@@ -89,6 +90,11 @@ server {
     add_header Referrer-Policy "no-referrer-when-downgrade";
     proxy_hide_header Referrer-Policy;
 
+    # CSP headers for the admin/dashboard resources
+<% if ( endpoint === 'admin' ) { -%>
+    add_header Content-Security-Policy "default-src 'none'; connect-src wss: https: 'self' *.cloudron.io; script-src https: 'self' 'unsafe-inline' 'unsafe-eval'; img-src * data:; style-src https: 'unsafe-inline'; object-src 'none'; font-src https: 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self';";
+<% } -%>
+
     proxy_http_version 1.1;
     proxy_intercept_errors on;
     proxy_read_timeout       3500;
@@ -106,7 +112,7 @@ server {
     proxy_set_header Connection $connection_upgrade;
 
     # only serve up the status page if we get proxy gateway errors
-    root <%= sourceDir %>/webadmin/dist;
+    root <%= sourceDir %>/dashboard/dist;
     error_page 502 503 504 /appstatus.html;
     location /appstatus.html {
         internal;
@@ -160,7 +166,7 @@ server {
         # }
 
         location / {
-            root   <%= sourceDir %>/webadmin/dist;
+            root   <%= sourceDir %>/dashboard/dist;
             index  index.html index.htm;
         }
 <% } else if ( endpoint === 'app' ) { %>

src/addons.js

@@ -291,7 +291,7 @@ function setupEmail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    mail.getAll(function (error, mailDomains) {
+    mail.getDomains(function (error, mailDomains) {
         if (error) return callback(error);
 
         const mailInDomains = mailDomains.filter(function (d) { return d.enabled; }).map(function (d) { return d.domain; }).join(',');

src/appdb.js

@@ -220,8 +220,8 @@ function add(id, appStoreId, manifest, location, domain, portBindings, data, cal
     // only allocate a mailbox if mailboxName is set
     if (data.mailboxName) {
         queries.push({
-            query: 'INSERT INTO mailboxes (name, domain, ownerId, ownerType) VALUES (?, ?, ?, ?)',
-            args: [ data.mailboxName, domain, id, mailboxdb.TYPE_APP ]
+            query: 'INSERT INTO mailboxes (name, type, domain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?)',
+            args: [ data.mailboxName, mailboxdb.TYPE_MAILBOX, domain, id, mailboxdb.OWNER_TYPE_APP ]
         });
     }
 

src/apps.js

@@ -172,14 +172,14 @@ function validatePortBindings(portBindings, tcpPorts) {
         993, /* imaps */
         2003, /* graphite (lo) */
         2004, /* graphite (lo) */
-        2020, /* install server */
+        2020, /* mail server */
         config.get('port'), /* app server (lo) */
         config.get('sysadminPort'), /* sysadmin app server (lo) */
         config.get('smtpPort'), /* internal smtp port (lo) */
         config.get('ldapPort'), /* ldap server (lo) */
         3306, /* mysql (lo) */
         4190, /* managesieve */
-        8000 /* graphite (lo) */
+        8000, /* graphite (lo) */
     ];
 
     if (!portBindings) return null;
@@ -901,7 +901,7 @@ function clone(appId, data, auditSource, callback) {
                 var newAppId = uuid.v4(), manifest = backupInfo.manifest;
 
                 appstore.purchase(newAppId, app.appStoreId, function (error) {
-                    if (error && error.reason === AppstoreError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+                    if (error && error.reason === AppstoreError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, error.message));
                     if (error && error.reason === AppstoreError.BILLING_REQUIRED) return callback(new AppsError(AppsError.BILLING_REQUIRED, error.message));
                     if (error && error.reason === AppstoreError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
                     if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));

src/appstore.js

@@ -164,7 +164,7 @@ function sendAliveStatus(callback) {
             });
         },
         function (callback) {
-            mail.getAll(function (error, result) {
+            mail.getDomains(function (error, result) {
                 if (error) return callback(new AppstoreError(AppstoreError.INTERNAL_ERROR, error));
                 mailDomains = result;
                 callback();

src/backups.js

@@ -232,7 +232,7 @@ function sync(backupConfig, backupId, dataDir, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     function setBackupProgress(message) {
-        debug(message);
+        debug('%s: %s', (new Date()).toISOString(), message);
         safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, message);
     }
 
@@ -257,13 +257,19 @@ function sync(backupConfig, backupId, dataDir, callback) {
             ++retryCount;
             debug(`${task.operation} ${task.path} try ${retryCount}`);
             if (task.operation === 'add') {
-                setBackupProgress(`Adding ${task.path}`);
+                setBackupProgress(`Adding ${task.path} position ${task.position} try ${retryCount}`);
                 var stream = fs.createReadStream(path.join(dataDir, task.path));
-                stream.on('error', function (error) { setBackupProgress(`read stream error for ${task.path}: ${error.message}`); retryCallback(); }); // ignore error if file disappears
-                api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, retryCallback);
+                stream.on('error', function (error) {
+                    setBackupProgress(`read stream error for ${task.path}: ${error.message}`);
+                    retryCallback();
+                }); // ignore error if file disappears
+                api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
+                    setBackupProgress(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
+                    retryCallback(error);
+                });
             }
         }, iteratorCallback);
-    }, 10 /* concurrency */, function (error) {
+    }, backupConfig.syncConcurrency || 10 /* concurrency */, function (error) {
         if (error) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
 
         callback();

src/backuptask.js

@@ -44,9 +44,9 @@ initialize(function (error) {
     safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, '');
 
     backups.upload(backupId, format, dataDir, function resultHandler(error) {
-        if (error) debug('completed with error', error);
+        if (error) debug('upload completed with error', error);
 
-        debug('completed');
+        debug('upload completed');
 
         safe.fs.writeFileSync(paths.BACKUP_RESULT_FILE, error ? error.message : '');
 

src/cloudron.js

@@ -234,30 +234,11 @@ function updateToLatest(auditSource, callback) {
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
     if (!boxUpdateInfo.sourceTarballUrl) return callback(new CloudronError(CloudronError.BAD_STATE, 'No automatic update available'));
 
-    // check if this is just a version number change
-    if (config.version().match(/[-+]/) !== null && config.version().replace(/[-+].*/, '') === boxUpdateInfo.version) {
-        doShortCircuitUpdate(boxUpdateInfo, function (error) {
-            if (error) debug('Short-circuit update failed', error);
-        });
-
-        return callback(null);
-    }
-
     if (boxUpdateInfo.upgrade && config.provider() !== 'caas') return callback(new CloudronError(CloudronError.SELF_UPGRADE_NOT_SUPPORTED));
 
     update(boxUpdateInfo, auditSource, callback);
 }
 
-function doShortCircuitUpdate(boxUpdateInfo, callback) {
-    assert(boxUpdateInfo !== null && typeof boxUpdateInfo === 'object');
-
-    debug('Starting short-circuit from prerelease version %s to release version %s', config.version(), boxUpdateInfo.version);
-    config.setVersion(boxUpdateInfo.version);
-    progress.clear(progress.UPDATE);
-    updateChecker.resetUpdateInfo();
-    callback();
-}
-
 function doUpdate(boxUpdateInfo, callback) {
     assert(boxUpdateInfo && typeof boxUpdateInfo === 'object');
 

src/database.js

@@ -6,10 +6,6 @@ exports = module.exports = {
     query: query,
     transaction: transaction,
 
-    beginTransaction: beginTransaction,
-    rollback: rollback,
-    commit: commit,
-
     importFromFile: importFromFile,
     exportToFile: exportToFile,
 
@@ -27,21 +23,13 @@ var assert = require('assert'),
 var gConnectionPool = null,
     gDefaultConnection = null;
 
-function initialize(options, callback) {
-    if (typeof options === 'function') {
-        callback = options;
-        options = {
-            connectionLimit: 5
-        };
-    }
-
-    assert.strictEqual(typeof options.connectionLimit, 'number');
+function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     if (gConnectionPool !== null) return callback(null);
 
     gConnectionPool  = mysql.createPool({
-        connectionLimit: options.connectionLimit,
+        connectionLimit: 5, // this has to be > 1 since we store one connection as 'default'. the rest for transactions
         host: config.database().hostname,
         user: config.database().username,
         password: config.database().password,

src/docker.js

@@ -45,6 +45,7 @@ var addons = require('./addons.js'),
     debug = require('debug')('box:docker.js'),
     once = require('once'),
     safe = require('safetydance'),
+    shell = require('./shell.js'),
     spawn = child_process.spawn,
     util = require('util'),
     _ = require('underscore');
@@ -58,42 +59,24 @@ function debugApp(app, args) {
 function pullImage(manifest, callback) {
     var docker = exports.connection;
 
-    docker.pull(manifest.dockerImage, function (err, stream) {
-        if (err) return callback(new Error('Error connecting to docker. statusCode: ' + err.statusCode));
+    // Use docker CLI here to support downloading of private repos. for dockerode, we have to use
+    // https://github.com/apocas/dockerode#pull-from-private-repos
+    shell.exec('pullImage', '/usr/bin/docker', [ 'pull', manifest.dockerImage ], { }, function (error) {
+        if (error) {
+            debug(`pullImage: Error pulling image ${manifest.dockerImage} of ${manifest.id}: ${error.message}`);
+            return callback(new Error('Failed to pull image'));
+        }
 
-        // https://github.com/dotcloud/docker/issues/1074 says each status message
-        // is emitted as a chunk
-        stream.on('data', function (chunk) {
-            var data = safe.JSON.parse(chunk) || { };
-            debug('pullImage %s: %j', manifest.id, data);
+        var image = docker.getImage(manifest.dockerImage);
 
-            // The information here is useless because this is per layer as opposed to per image
-            if (data.status) {
-            } else if (data.error) {
-                debug('pullImage error %s: %s', manifest.id, data.errorDetail.message);
-            }
-        });
+        image.inspect(function (err, data) {
+            if (err) return callback(new Error('Error inspecting image:' + err.message));
+            if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
+            if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
 
-        stream.on('end', function () {
-            debug('downloaded image %s of %s successfully', manifest.dockerImage, manifest.id);
+            if (data.Config.ExposedPorts) debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
 
-            var image = docker.getImage(manifest.dockerImage);
-
-            image.inspect(function (err, data) {
-                if (err) return callback(new Error('Error inspecting image:' + err.message));
-                if (!data || !data.Config) return callback(new Error('Missing Config in image:' + JSON.stringify(data, null, 4)));
-                if (!data.Config.Entrypoint && !data.Config.Cmd) return callback(new Error('Only images with entry point are allowed'));
-
-                if (data.Config.ExposedPorts) debug('This image of %s exposes ports: %j', manifest.id, data.Config.ExposedPorts);
-
-                callback(null);
-            });
-        });
-
-        stream.on('error', function (error) {
-            debug('error pulling image %s of %s: %j', manifest.dockerImage, manifest.id, error);
-
-            callback(error);
+            callback(null);
         });
     });
 }

src/eventlog.js

@@ -121,14 +121,7 @@ function cleanup(callback) {
     var d = new Date();
     d.setDate(d.getDate() - 10); // 10 days ago
 
-    // only cleanup high frequency events
-    var actions = [
-        exports.ACTION_USER_LOGIN,
-        exports.ACTION_BACKUP_START,
-        exports.ACTION_BACKUP_FINISH
-    ];
-
-    eventlogdb.delByCreationTime(d, actions, function (error) {
+    eventlogdb.delByCreationTime(d, function (error) {
         if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
 
         callback(null);

src/eventlogdb.js

@@ -121,15 +121,13 @@ function clear(callback) {
     });
 }
 
-function delByCreationTime(creationTime, actions, callback) {
+function delByCreationTime(creationTime, callback) {
     assert(util.isDate(creationTime));
-    assert(Array.isArray(actions));
     assert.strictEqual(typeof callback, 'function');
 
-    var query = 'DELETE FROM eventlog WHERE creationTime < ? ';
-    if (actions.length) query += ' AND ( ' + actions.map(function () { return 'action != ?'; }).join(' AND ') + ' ) ';
+    var query = 'DELETE FROM eventlog WHERE creationTime < ?';
 
-    database.query(query, [ creationTime ].concat(actions), function (error) {
+    database.query(query, [ creationTime ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(error);

src/groups.js

@@ -24,7 +24,6 @@ var assert = require('assert'),
     constants = require('./constants.js'),
     DatabaseError = require('./databaseerror.js'),
     groupdb = require('./groupdb.js'),
-    mailboxdb = require('./mailboxdb.js'),
     util = require('util'),
     uuid = require('uuid');
 
@@ -100,15 +99,11 @@ function remove(id, callback) {
     // never allow admin group to be deleted
     if (id === constants.ADMIN_GROUP_ID) return callback(new GroupError(GroupError.NOT_ALLOWED));
 
-    mailboxdb.delByOwnerId(id, function (error) {
+    groupdb.del(id, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
         if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
 
-        groupdb.del(id, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new GroupError(GroupError.NOT_FOUND));
-            if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
-
-            callback(null);
-        });
+        callback(null);
     });
 }
 

src/infra_version.js

@@ -18,7 +18,7 @@ exports = module.exports = {
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:1.0.0' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:1.0.1' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:1.0.0' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:1.2.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:1.2.2' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:1.0.0' }
     }
 };

src/ldap.js

@@ -430,11 +430,11 @@ function authenticateMailbox(req, res, next) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.message));
 
-        mail.get(parts[1], function (error, domain) {
+        mail.getDomain(parts[1], function (error, domain) {
             if (error && error.reason === MailError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
             if (error) return next(new ldap.OperationsError(error.message));
 
-            if (mailbox.ownerType === mailboxdb.TYPE_APP) {
+            if (mailbox.ownerType === mailboxdb.OWNER_TYPE_APP) {
                 var addonId = req.dn.rdns[1].attrs.ou.value.toLowerCase(); // 'sendmail' or 'recvmail'
                 var name;
                 if (addonId === 'sendmail') name = 'MAIL_SMTP_PASSWORD';
@@ -448,10 +448,10 @@ function authenticateMailbox(req, res, next) {
                     eventlog.add(eventlog.ACTION_APP_LOGIN, { authType: 'ldap', mailboxId: name }, { appId: mailbox.ownerId, addonId: addonId });
                     return res.end();
                 });
-            } else if (mailbox.ownerType === mailboxdb.TYPE_USER) {
+            } else if (mailbox.ownerType === mailboxdb.OWNER_TYPE_USER) {
                 if (!domain.enabled) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-                user.verifyWithUsername(parts[0], req.credentials || '', function (error, result) {
+                user.verify(mailbox.ownerId, req.credentials || '', function (error, result) {
                     if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
                     if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
                     if (error) return next(new ldap.OperationsError(error.message));

src/mail.js

@@ -3,12 +3,12 @@
 exports = module.exports = {
     getStatus: getStatus,
 
-    get: get,
-    getAll: getAll,
+    getDomains: getDomains,
 
-    add: add,
-    del: del,
-    update: update,
+    getDomain: getDomain,
+    addDomain: addDomain,
+    removeDomain: removeDomain,
+    updateDomain: updateDomain,
 
     addDnsRecords: addDnsRecords,
 
@@ -23,16 +23,19 @@ exports = module.exports = {
 
     getMailboxes: getMailboxes,
     removeMailboxes: removeMailboxes,
-    getUserMailbox: getUserMailbox,
-    enableUserMailbox: enableUserMailbox,
-    disableUserMailbox: disableUserMailbox,
+    getMailbox: getMailbox,
+    addMailbox: addMailbox,
+    updateMailbox: updateMailbox,
+    removeMailbox: removeMailbox,
 
+    listAliases: listAliases,
     getAliases: getAliases,
     setAliases: setAliases,
 
     getLists: getLists,
     getList: getList,
     addList: addList,
+    updateList: updateList,
     removeList: removeList,
 
     _readDkimPublicKeySync: readDkimPublicKeySync,
@@ -48,8 +51,6 @@ var assert = require('assert'),
     debug = require('debug')('box:mail'),
     dns = require('./native-dns.js'),
     domains = require('./domains.js'),
-    groups = require('./groups.js'),
-    GroupError = groups.GroupError,
     infra = require('./infra_version.js'),
     mailboxdb = require('./mailboxdb.js'),
     maildb = require('./maildb.js'),
@@ -65,7 +66,6 @@ var assert = require('assert'),
     smtpTransport = require('nodemailer-smtp-transport'),
     sysinfo = require('./sysinfo.js'),
     user = require('./user.js'),
-    UserError = user.UserError,
     util = require('util'),
     _ = require('underscore');
 
@@ -97,19 +97,19 @@ MailError.ALREADY_EXISTS = 'Already Exists';
 MailError.NOT_FOUND = 'Not Found';
 MailError.IN_USE = 'In Use';
 
-function validateAlias(alias) {
-    assert.strictEqual(typeof alias, 'string');
+function validateName(name) {
+    assert.strictEqual(typeof name, 'string');
 
-    if (alias.length < 1) return new MailError(MailError.BAD_FIELD, 'alias must be atleast 1 char');
-    if (alias.length >= 200) return new MailError(MailError.BAD_FIELD, 'alias too long');
+    if (name.length < 1) return new MailError(MailError.BAD_FIELD, 'mailbox name must be atleast 1 char');
+    if (name.length >= 200) return new MailError(MailError.BAD_FIELD, 'mailbox name too long');
 
-    if (constants.RESERVED_NAMES.indexOf(alias) !== -1) return new MailError(MailError.BAD_FIELD, 'alias is reserved');
+    if (constants.RESERVED_NAMES.indexOf(name) !== -1) return new MailError(MailError.BAD_FIELD, `mailbox name ${name} is reserved`);
 
     // +/- can be tricky in emails. also need to consider valid LDAP characters here (e.g '+' is reserved)
-    if (/[^a-zA-Z0-9.]/.test(alias)) return new MailError(MailError.BAD_FIELD, 'alias can only contain alphanumerals and dot');
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailError(MailError.BAD_FIELD, 'mailbox name can only contain alphanumerals and dot');
 
     // app emails are sent using the .app suffix
-    if (alias.indexOf('.app') !== -1) return new MailError(MailError.BAD_FIELD, 'alias pattern is reserved for apps');
+    if (name.indexOf('.app') !== -1) return new MailError(MailError.BAD_FIELD, 'mailbox name pattern is reserved for apps');
 
     return null;
 }
@@ -445,7 +445,7 @@ function getStatus(domain, callback) {
         };
     }
 
-    get(domain, function (error, result) {
+    getDomain(domain, function (error, result) {
         if (error) return callback(error);
 
         var checks = [
@@ -477,7 +477,7 @@ function createMailConfig(callback) {
 
     debug('createMailConfig: generating mail config');
 
-    maildb.getAll(function (error, mailDomains) {
+    getDomains(function (error, mailDomains) {
         if (error) return callback(error);
 
         user.getOwner(function (error, owner) {
@@ -571,6 +571,7 @@ function restartMail(callback) {
                         -v "${paths.MAIL_DATA_DIR}:/app/data" \
                         -v "${paths.PLATFORM_DATA_DIR}/addons/mail:/etc/mail" \
                         ${ports} \
+                        -p 127.0.0.1:2020:2020 \
                         --read-only -v /run -v /tmp ${tag}`;
 
             shell.execSync('startMail', cmd);
@@ -580,7 +581,7 @@ function restartMail(callback) {
     });
 }
 
-function get(domain, callback) {
+function getDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
@@ -592,7 +593,7 @@ function get(domain, callback) {
     });
 }
 
-function getAll(callback) {
+function getDomains(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     maildb.getAll(function (error, results) {
@@ -724,7 +725,7 @@ function addDnsRecords(domain, callback) {
     });
 }
 
-function add(domain, callback) {
+function addDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
@@ -733,18 +734,21 @@ function add(domain, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'No such domain'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-        addDnsRecords(domain, NOOP_CALLBACK); // add the required dns records asynchronously
+        async.series([
+            addDnsRecords.bind(null, domain), // do this first to ensure DKIM keys
+            restartMail
+        ], NOOP_CALLBACK); // do these asynchronously
 
         callback();
     });
 }
 
 // this is just a way to resync the mail "dns" records via the UI
-function update(domain, callback) {
+function updateDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    get(domain, function (error) {
+    getDomain(domain, function (error) {
         if (error) return callback(error);
 
         addDnsRecords(domain, NOOP_CALLBACK);
@@ -753,22 +757,18 @@ function update(domain, callback) {
     });
 }
 
-function del(domain, callback) {
+function removeDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    get(domain, function (error, result) {
-        if (error) return callback(error);
+    maildb.del(domain, function (error) {
+        if (error && error.reason === DatabaseError.IN_USE) return callback(new MailError(MailError.IN_USE));
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, error.message));
+        if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-        maildb.del(domain, function (error) {
-            if (error && error.reason === DatabaseError.IN_USE) return callback(new MailError(MailError.IN_USE));
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, error.message));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
+        restartMail(NOOP_CALLBACK);
 
-            if (result && result.enabled) restartMail(NOOP_CALLBACK);
-
-            callback();
-        });
+        callback();
     });
 }
 
@@ -787,12 +787,12 @@ function setMailFromValidation(domain, enabled, callback) {
     });
 }
 
-function setCatchAllAddress(domain, address, callback) {
+function setCatchAllAddress(domain, addresses, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert(Array.isArray(address));
+    assert(Array.isArray(addresses));
     assert.strictEqual(typeof callback, 'function');
 
-    maildb.update(domain, { catchAll: address }, function (error) {
+    maildb.update(domain, { catchAll: addresses }, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
@@ -852,7 +852,7 @@ function sendTestMail(domain, to, callback) {
     assert.strictEqual(typeof to, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    get(domain, function (error, result) {
+    getDomain(domain, function (error, result) {
         if (error) return callback(error);
 
         mailer.sendTestMail(result.domain, to);
@@ -883,74 +883,91 @@ function removeMailboxes(domain, callback) {
     });
 }
 
-function getUserMailbox(domain, userId, callback) {
+function getMailbox(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    user.get(userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such user'));
+    mailboxdb.getMailbox(name, domain, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-        if (!result.username) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
 
-        mailboxdb.getMailbox(result.username, domain, function (error, result) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            callback(null, result);
-        });
+        callback(null, result);
     });
 }
 
-function enableUserMailbox(domain, userId, callback) {
+function addMailbox(name, domain, userId, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    user.get(userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such user'));
-        if (error) return callback(new MailError(MailError.INTERNAL_ERROR));
-        if (!result.username) return callback(new MailError(MailError.NOT_FOUND, 'user has no username'));
+    name = name.toLowerCase();
 
-        mailboxdb.add(result.username, domain, userId, mailboxdb.TYPE_USER, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailError(MailError.ALREADY_EXISTS, 'mailbox already exists'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
+    var error = validateName(name);
+    if (error) return callback(error);
 
-            callback(null);
-        });
+    mailboxdb.addMailbox(name, domain, userId, mailboxdb.OWNER_TYPE_USER, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailError(MailError.ALREADY_EXISTS, `mailbox ${name} already exists`));
+        if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
+
+        callback(null);
     });
 }
 
-function disableUserMailbox(domain, userId, callback) {
+function updateMailbox(name, domain, userId, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    user.get(userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such user'));
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.updateMailbox(name, domain, userId, mailboxdb.OWNER_TYPE_USER, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-        if (!result.username) return callback(new MailError(MailError.NOT_FOUND, 'user has no username'));
 
-        mailboxdb.del(result.username, domain, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            callback(null);
-        });
+        callback(null);
     });
 }
 
-function getAliases(domain, userId, callback) {
+function removeMailbox(name, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    user.get(userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such user'));
+    mailboxdb.del(name, domain, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-        if (!result.username) return callback(new MailError(MailError.NOT_FOUND, 'user has no username'));
 
-        mailboxdb.getAliasesForName(result.username, domain, function (error, aliases) {
+        callback(null);
+    });
+}
+
+function listAliases(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.listAliases(domain, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, error.message));
+        if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
+
+        callback(null, result);
+    });
+}
+
+function getAliases(name, domain, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getMailbox(name, domain, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.getAliasesForName(name, domain, function (error, aliases) {
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
             if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
@@ -959,31 +976,30 @@ function getAliases(domain, userId, callback) {
     });
 }
 
-function setAliases(domain, userId, aliases, callback) {
+function setAliases(name, domain, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof userId, 'string');
     assert(Array.isArray(aliases));
     assert.strictEqual(typeof callback, 'function');
 
     for (var i = 0; i < aliases.length; i++) {
         aliases[i] = aliases[i].toLowerCase();
 
-        var error = validateAlias(aliases[i]);
+        var error = validateName(aliases[i]);
         if (error) return callback(error);
     }
 
-    user.get(userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such user'));
+    mailboxdb.setAliasesForName(name, domain, aliases, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS && error.message.indexOf('mailboxes_name_domain_unique_index') !== -1) {
+            var aliasMatch = error.message.match(new RegExp(`^ER_DUP_ENTRY: Duplicate entry '(.*)-${domain}' for key 'mailboxes_name_domain_unique_index'$`))
+            if (!aliasMatch) return callback(new MailError(MailError.ALREADY_EXISTS, error.message));
+            return callback(new MailError(MailError.ALREADY_EXISTS, `Mailbox, mailinglist or alias for ${aliasMatch[1]} already exists`));
+        }
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailError(MailError.ALREADY_EXISTS, error.message));
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-        if (!result.username) return callback(new MailError(MailError.NOT_FOUND, 'user has no username'));
 
-        mailboxdb.setAliasesForName(result.username, domain, aliases, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailError(MailError.ALREADY_EXISTS, error.message));
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            callback(null);
-        });
+        callback(null);
     });
 }
 
@@ -998,56 +1014,80 @@ function getLists(domain, callback) {
     });
 }
 
-function getList(domain, groupId, callback) {
+function getList(domain, listName, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof listName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    groups.get(groupId, function (error, result) {
-        if (error && error.reason === GroupError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such group'));
+    mailboxdb.getGroup(listName, domain, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such list'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-        mailboxdb.getGroup(result.name, domain, function (error, result) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such list'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            callback(null, result);
-        });
+        callback(null, result);
     });
 }
 
-function addList(domain, groupId, callback) {
+function addList(name, domain, members, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof groupId, 'string');
+    assert.strictEqual(typeof name, 'string');
+    assert(Array.isArray(members));
     assert.strictEqual(typeof callback, 'function');
 
-    groups.get(groupId, function (error, result) {
-        if (error && error.reason === GroupError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such group'));
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    for (var i = 0; i < members.length; i++) {
+        members[i] = members[i].toLowerCase();
+
+        error = validateName(members[i]);
+        if (error) return callback(error);
+    }
+
+    mailboxdb.addGroup(name, domain, members, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailError(MailError.ALREADY_EXISTS, 'list already exits'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-        mailboxdb.add(result.name, domain, groupId, mailboxdb.TYPE_GROUP, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailError(MailError.ALREADY_EXISTS, 'list already exits'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            callback();
-        });
+        callback();
     });
 }
 
-function removeList(domain, groupId, callback) {
+function updateList(name, domain, members, callback) {
+    assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof groupId, 'string');
+    assert(Array.isArray(members));
     assert.strictEqual(typeof callback, 'function');
 
-    groups.get(groupId, function (error, result) {
-        if (error && error.reason === GroupError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such group'));
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    for (var i = 0; i < members.length; i++) {
+        members[i] = members[i].toLowerCase();
+
+        error = validateName(members[i]);
+        if (error) return callback(error);
+    }
+
+    mailboxdb.updateList(name, domain, members, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such mailbox'));
         if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
 
-        mailboxdb.del(result.name, domain, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such list'));
-            if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
-
-            callback();
-        });
+        callback(null);
+    });
+}
+
+function removeList(domain, listName, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof listName, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.del(listName, domain, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailError(MailError.NOT_FOUND, 'no such list'));
+        if (error) return callback(new MailError(MailError.INTERNAL_ERROR, error));
+
+        callback();
     });
 }

src/mailboxdb.js

@@ -1,7 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    add: add,
+    addMailbox: addMailbox,
+    addGroup: addGroup,
+
+    updateMailbox: updateMailbox,
+    updateList: updateList,
     del: del,
 
     listAliases: listAliases,
@@ -23,26 +27,38 @@ exports = module.exports = {
 
     _clear: clear,
 
-    TYPE_USER: 'user',
-    TYPE_APP: 'app',
-    TYPE_GROUP: 'group'
+    TYPE_MAILBOX: 'mailbox',
+    TYPE_LIST: 'list',
+    TYPE_ALIAS: 'alias',
+
+    OWNER_TYPE_USER: 'user',
+    OWNER_TYPE_APP: 'app',
+    OWNER_TYPE_GROUP: 'group' // obsolete
 };
 
 var assert = require('assert'),
     database = require('./database.js'),
     DatabaseError = require('./databaseerror.js'),
+    safe = require('safetydance'),
     util = require('util');
 
-var MAILBOX_FIELDS = [ 'name', 'ownerId', 'ownerType', 'aliasTarget', 'creationTime', 'domain' ].join(',');
+var MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'ownerType', 'aliasTarget', 'creationTime', 'membersJson', 'domain' ].join(',');
 
-function add(name, domain, ownerId, ownerType, callback) {
+function postProcess(data) {
+    data.members = safe.JSON.parse(data.membersJson) || [ ];
+    delete data.membersJson;
+
+    return data;
+}
+
+function addMailbox(name, domain, ownerId, ownerType, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof ownerId, 'string');
     assert.strictEqual(typeof ownerType, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('INSERT INTO mailboxes (name, domain, ownerId, ownerType) VALUES (?, ?, ?, ?)', [ name, domain, ownerId, ownerType ], function (error) {
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType) VALUES (?, ?, ?, ?, ?)', [ name, exports.TYPE_MAILBOX, domain, ownerId, ownerType ], function (error) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, 'mailbox already exists'));
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
@@ -50,6 +66,51 @@ function add(name, domain, ownerId, ownerType, callback) {
     });
 }
 
+function updateMailbox(name, domain, ownerId, ownerType, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof ownerId, 'string');
+    assert.strictEqual(typeof ownerType, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('UPDATE mailboxes SET ownerId = ? WHERE name = ? AND domain = ? AND ownerType = ?', [ ownerId, name, domain, ownerType ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function addGroup(name, domain, members, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert(Array.isArray(members));
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, ownerType, membersJson) VALUES (?, ?, ?, ?, ?, ?)',
+        [ name, exports.TYPE_LIST, domain, 'admin', exports.OWNER_TYPE_GROUP, JSON.stringify(members) ], function (error) {
+            if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, 'mailbox already exists'));
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
+}
+
+function updateList(name, domain, members, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert(Array.isArray(members));
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('UPDATE mailboxes SET membersJson = ? WHERE name = ? AND domain = ?',
+        [ JSON.stringify(members), name, domain ], function (error, result) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+            callback(null);
+        });
+}
+
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -77,7 +138,6 @@ function delByDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    // deletes aliases as well
     database.query('DELETE FROM mailboxes WHERE domain = ?', [ domain ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
@@ -89,7 +149,6 @@ function delByOwnerId(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    // deletes aliases as well
     database.query('DELETE FROM mailboxes WHERE ownerId=?', [ id ], function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
@@ -121,35 +180,41 @@ function getMailbox(name, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ? AND (ownerType = ? OR ownerType = ?) AND aliasTarget IS NULL', [ name, domain, exports.TYPE_APP, exports.TYPE_USER ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND type = ? AND domain = ?',
+        [ name, exports.TYPE_MAILBOX, domain ], function (error, results) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, results[0]);
-    });
+            callback(null, postProcess(results[0]));
+        });
 }
 
 function listMailboxes(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE domain = ? AND (ownerType = ? OR ownerType = ?) AND aliasTarget IS NULL ORDER BY name', [ domain, exports.TYPE_APP, exports.TYPE_USER ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND domain = ? ORDER BY name',
+        [ exports.TYPE_MAILBOX, domain ], function (error, results) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, results);
-    });
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
 }
 
 function listGroups(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE domain = ? AND ownerType = ? AND aliasTarget IS NULL', [ domain, exports.TYPE_GROUP ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND domain = ?',
+        [ exports.TYPE_LIST, domain ], function (error, results) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, results);
-    });
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
 }
 
 function getGroup(name, domain, callback) {
@@ -157,25 +222,13 @@ function getGroup(name, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    // This can be merged into a single query but cannot get 'not found' information
-    // SELECT users.username FROM mailboxes
-    //    INNER JOIN groupMembers ON mailboxes.ownerId = groupMembers.groupId
-    //    INNER JOIN users ON groupMembers.userId = users.id
-    //    WHERE mailboxes.name = <name>
-
-    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ? AND ownerType = ? AND aliasTarget IS NULL', [ name, domain, exports.TYPE_GROUP ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
-
-        // username can be null if the user has not signed up with the invite yet
-        database.query('SELECT users.username FROM groupMembers INNER JOIN users ON groupMembers.userId = users.id WHERE groupMembers.groupId = ? AND users.username IS NOT NULL', [ results[0].ownerId ], function (error, memberList) {
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND name = ? AND domain = ?',
+        [ exports.TYPE_LIST, name, domain ], function (error, results) {
             if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-            results[0].members = memberList.map(function (m) { return m.username; });
-
-            callback(null, results[0]);
+            callback(null, postProcess(results[0]));
         });
-    });
 }
 
 function getByOwnerId(ownerId, callback) {
@@ -186,6 +239,8 @@ function getByOwnerId(ownerId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
+        results.forEach(function (result) { postProcess(result); });
+
         callback(null, results);
     });
 }
@@ -201,10 +256,11 @@ function setAliasesForName(name, domain, aliases, callback) {
         if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
         var queries = [];
-        queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ? AND domain = ?', args: [ name, domain ] });
+        // clear existing aliases
+        queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ? AND domain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
         aliases.forEach(function (alias) {
-            queries.push({ query: 'INSERT INTO mailboxes (name, domain, aliasTarget, ownerId, ownerType) VALUES (?, ?, ?, ?, ?)',
-                args: [ alias, domain, name, results[0].ownerId, results[0].ownerType ] });
+            queries.push({ query: 'INSERT INTO mailboxes (name, type, domain, aliasTarget, ownerId, ownerType) VALUES (?, ?, ?, ?, ?, ?)',
+                args: [ alias, exports.TYPE_ALIAS, domain, name, results[0].ownerId, results[0].ownerType ] });
         });
 
         database.transaction(queries, function (error) {
@@ -221,23 +277,27 @@ function getAliasesForName(name, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT name FROM mailboxes WHERE aliasTarget = ? AND domain = ? ORDER BY name', [ name, domain ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.query('SELECT name FROM mailboxes WHERE type = ? AND aliasTarget = ? AND domain = ? ORDER BY name',
+        [ exports.TYPE_ALIAS, name, domain ], function (error, results) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        results = results.map(function (r) { return r.name; });
-        callback(null, results);
-    });
+            results = results.map(function (r) { return r.name; });
+            callback(null, results);
+        });
 }
 
 function listAliases(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE domain = ? AND aliasTarget IS NOT NULL ORDER BY name', [ domain ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE domain = ? AND type = ? ORDER BY name',
+        [ domain, exports.TYPE_ALIAS ], function (error, results) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        callback(null, results);
-    });
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
 }
 
 function getAlias(name, domain, callback) {
@@ -245,10 +305,13 @@ function getAlias(name, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND domain = ? AND aliasTarget IS NOT NULL', [ name, domain ], function (error, results) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE name = ? AND type = ? AND domain = ?',
+        [ name, exports.TYPE_ALIAS, domain ], function (error, results) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, results[0]);
-    });
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results[0]);
+        });
 }

src/mailer.js

@@ -74,7 +74,7 @@ function getMailConfig(callback) {
                 cloudronName = 'Cloudron';
             }
 
-            mail.getAll(function (error, domains) {
+            mail.getDomains(function (error, domains) {
                 if (error) return callback(error);
                 if (domains.length === 0) return callback('No domains configured');
 

src/progress.js

@@ -41,7 +41,7 @@ function setDetail(tag, detail) {
     assert.strictEqual(typeof tag, 'string');
     assert.strictEqual(typeof detail, 'string');
 
-    if (!progress[tag]) return debug('unable to set detail %s', detail);
+    if (!progress[tag]) return debug('[%s] %s', tag, detail);
 
     progress[tag].detail = detail;
 }

src/routes/domains.js

@@ -93,7 +93,7 @@ function del(req, res, next) {
 
     domains.del(req.params.domain, function (error) {
         if (error && error.reason === DomainError.NOT_FOUND) return next(new HttpError(404, error.message));
-        if (error && error.reason === DomainError.IN_USE) return next(new HttpError(409, 'Domain is still in use. Remove all apps using this domain'));
+        if (error && error.reason === DomainError.IN_USE) return next(new HttpError(409, 'Domain is still in use. Remove all apps and mailboxes using this domain'));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(204));

src/routes/mail.js

@@ -1,11 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    get: get,
-
-    add: add,
-    update: update,
-    del: del,
+    getDomain: getDomain,
+    addDomain: addDomain,
+    getDomainStats: getDomainStats,
+    updateDomain: updateDomain,
+    removeDomain: removeDomain,
 
     getStatus: getStatus,
 
@@ -17,16 +17,19 @@ exports = module.exports = {
     sendTestMail: sendTestMail,
 
     getMailboxes: getMailboxes,
-    getUserMailbox: getUserMailbox,
-    enableUserMailbox: enableUserMailbox,
-    disableUserMailbox: disableUserMailbox,
+    getMailbox: getMailbox,
+    addMailbox: addMailbox,
+    updateMailbox: updateMailbox,
+    removeMailbox: removeMailbox,
 
+    listAliases: listAliases,
     getAliases: getAliases,
     setAliases: setAliases,
 
     getLists: getLists,
     getList: getList,
     addList: addList,
+    updateList: updateList,
     removeList: removeList
 };
 
@@ -34,12 +37,16 @@ var assert = require('assert'),
     mail = require('../mail.js'),
     MailError = mail.MailError,
     HttpError = require('connect-lastmile').HttpError,
-    HttpSuccess = require('connect-lastmile').HttpSuccess;
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    middleware = require('../middleware/index.js'),
+    url = require('url');
 
-function get(req, res, next) {
+var mailProxy = middleware.proxy(url.parse('http://127.0.0.1:2020'));
+
+function getDomain(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
-    mail.get(req.params.domain, function (error, result) {
+    mail.getDomain(req.params.domain, function (error, result) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -47,12 +54,12 @@ function get(req, res, next) {
     });
 }
 
-function add(req, res, next) {
+function addDomain(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
 
-    mail.add(req.body.domain, function (error) {
+    mail.addDomain(req.body.domain, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error && error.reason === MailError.ALREADY_EXISTS) return next(new HttpError(409, 'domain already exists'));
         if (error) return next(new HttpError(500, error));
@@ -61,11 +68,24 @@ function add(req, res, next) {
     });
 }
 
-function update(req, res, next) {
+function getDomainStats(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+
+    var parsedUrl = url.parse(req.url, true /* parseQueryString */);
+    delete parsedUrl.query['access_token'];
+    delete req.headers['authorization'];
+    delete req.headers['cookies'];
+
+    req.url = url.format({ pathname: req.params.domain, query: parsedUrl.query });
+
+    mailProxy(req, res, next);
+}
+
+function updateDomain(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.domain, 'string');
 
-    mail.update(req.params.domain, function (error) {
+    mail.updateDomain(req.params.domain, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -73,10 +93,10 @@ function update(req, res, next) {
     });
 }
 
-function del(req, res, next) {
+function removeDomain(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
-    mail.del(req.params.domain, function (error) {
+    mail.removeDomain(req.params.domain, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error && error.reason === MailError.IN_USE) return next(new HttpError(409, 'Mail domain is still in use. Remove existing mailboxes'));
         if (error) return next(new HttpError(500, error));
@@ -118,13 +138,14 @@ function setCatchAllAddress(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.body, 'object');
 
-    if (!req.body.address || !Array.isArray(req.body.address)) return next(new HttpError(400, 'address array is required'));
+    if (!req.body.addresses) return next(new HttpError(400, 'addresses is required'));
+    if (!Array.isArray(req.body.addresses)) return next(new HttpError(400, 'addresses must be an array of strings'));
 
-    for (var i = 0; i < req.body.address.length; i++) {
-        if (typeof req.body.address[i] !== 'string') return next(new HttpError(400, 'address must be an array of string'));
+    for (var i = 0; i < req.body.addresses.length; i++) {
+        if (typeof req.body.addresses[i] !== 'string') return next(new HttpError(400, 'addresses must be an array of strings'));
     }
 
-    mail.setCatchAllAddress(req.params.domain, req.body.address, function (error) {
+    mail.setCatchAllAddress(req.params.domain, req.body.addresses, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error && error.reason === MailError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
@@ -192,11 +213,11 @@ function getMailboxes(req, res, next) {
     });
 }
 
-function getUserMailbox(req, res, next) {
+function getMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.userId, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.getUserMailbox(req.params.domain, req.params.userId, function (error, result) {
+    mail.getMailbox(req.params.name, req.params.domain, function (error, result) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -204,36 +225,65 @@ function getUserMailbox(req, res, next) {
     });
 }
 
-function enableUserMailbox(req, res, next) {
+function addMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.userId, 'string');
 
-    mail.enableUserMailbox(req.params.domain, req.params.userId, function (error) {
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
+    if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
+
+    mail.addMailbox(req.body.name, req.params.domain, req.body.userId, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
-        if (error && error.reason === MailError.ALREADY_EXISTS) return next(new HttpSuccess(201, {}));
+        if (error && error.reason === MailError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
+        if (error && error.reason === MailError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(201, {}));
     });
 }
 
-function disableUserMailbox(req, res, next) {
+function updateMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.userId, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.disableUserMailbox(req.params.domain, req.params.userId, function (error) {
-        if (error && error.reason === MailError.NOT_FOUND) return next(new HttpSuccess(201, {}));
+    if (typeof req.body.userId !== 'string') return next(new HttpError(400, 'userId must be a string'));
+
+    mail.updateMailbox(req.params.name, req.params.domain, req.body.userId, function (error) {
+        if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === MailError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function removeMailbox(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
+
+    mail.removeMailbox(req.params.name, req.params.domain, function (error) {
+        if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(201, {}));
     });
 }
 
+function listAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+
+    mail.listAliases(req.params.domain, function (error, result) {
+        if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { aliases: result }));
+    });
+}
+
 function getAliases(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.userId, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.getAliases(req.params.domain, req.params.userId, function (error, result) {
+    mail.getAliases(req.params.name, req.params.domain, function (error, result) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -243,7 +293,7 @@ function getAliases(req, res, next) {
 
 function setAliases(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.userId, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
     assert.strictEqual(typeof req.body, 'object');
 
     if (!Array.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
@@ -252,8 +302,10 @@ function setAliases(req, res, next) {
         if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
     }
 
-    mail.setAliases(req.params.domain, req.params.userId, req.body.aliases, function (error) {
+    mail.setAliases(req.params.name, req.params.domain, req.body.aliases, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === MailError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
+        if (error && error.reason === MailError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202));
@@ -273,9 +325,9 @@ function getLists(req, res, next) {
 
 function getList(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.groupId, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.getList(req.params.domain, req.params.groupId, function (error, result) {
+    mail.getList(req.params.domain, req.params.name, function (error, result) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 
@@ -287,22 +339,47 @@ function addList(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.body, 'object');
 
-    if (typeof req.body.groupId !== 'string') return next(new HttpError(400, 'groupId must be a string'));
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be a string'));
+    if (!Array.isArray(req.body.members)) return next(new HttpError(400, 'members must be a string'));
 
-    mail.addList(req.params.domain, req.body.groupId, function (error) {
+    for (var i = 0; i < req.body.members.length; i++) {
+        if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
+    }
+
+    mail.addList(req.body.name, req.params.domain, req.body.members, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error && error.reason === MailError.ALREADY_EXISTS) return next(new HttpError(409, 'list already exists'));
+        if (error && error.reason === MailError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(201, {}));
     });
 }
 
+function updateList(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
+
+    if (!Array.isArray(req.body.members)) return next(new HttpError(400, 'members must be a string'));
+
+    for (var i = 0; i < req.body.members.length; i++) {
+        if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
+    }
+
+    mail.updateList(req.params.name, req.params.domain, req.body.members, function (error) {
+        if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === MailError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
 function removeList(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
-    assert.strictEqual(typeof req.params.groupId, 'string');
+    assert.strictEqual(typeof req.params.name, 'string');
 
-    mail.removeList(req.params.domain, req.params.groupId, function (error) {
+    mail.removeList(req.params.domain, req.params.name, function (error) {
         if (error && error.reason === MailError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
 

src/routes/settings.js

@@ -153,6 +153,10 @@ function setBackupConfig(req, res, next) {
     if (typeof req.body.provider !== 'string') return next(new HttpError(400, 'provider is required'));
     if (typeof req.body.retentionSecs !== 'number') return next(new HttpError(400, 'retentionSecs is required'));
     if ('key' in req.body && typeof req.body.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if ('syncConcurrency' in req.body) {
+        if (typeof req.body.syncConcurrency !== 'number') return next(new HttpError(400, 'syncConcurrency must be a positive integer'));
+        if (req.body.syncConcurrency < 1) return next(new HttpError(400, 'syncConcurrency must be a positive integer'));
+    }
     if (typeof req.body.format !== 'string') return next(new HttpError(400, 'format must be a string'));
     if ('acceptSelfSignedCerts' in req.body && typeof req.body.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
 

src/routes/test/mail-test.js

@@ -26,11 +26,10 @@ const DOMAIN_0 = {
     fallbackCertificate: null,
     tlsConfig: { provider: 'fallback' }
 };
-var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
-const GROUP_NAME = 'maillistgroup';
+const USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com', MAILBOX_NAME = 'superman';
+const LIST_NAME = 'devs';
 var token = null;
 var userId = '';
-var groupObject = null;
 
 function setup(done) {
     config._reset();
@@ -79,7 +78,7 @@ function cleanup(done) {
 }
 
 describe('Mail API', function () {
-    this.timeout(5000);
+    this.timeout(10000);
 
     before(setup);
     after(cleanup);
@@ -534,7 +533,7 @@ describe('Mail API', function () {
                 });
         });
 
-        it('cannot set without address field', function (done) {
+        it('cannot set without addresses field', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/catch_all')
                 .query({ access_token: token })
                 .end(function (err, res) {
@@ -543,10 +542,10 @@ describe('Mail API', function () {
                 });
         });
 
-        it('cannot set with bad address field', function (done) {
+        it('cannot set with bad addresses field', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/catch_all')
                 .query({ access_token: token })
-                .send({ address: [ 'user1', 123 ] })
+                .send({ addresses: [ 'user1', 123 ] })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
                     done();
@@ -556,7 +555,7 @@ describe('Mail API', function () {
         it('set succeeds', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/catch_all')
                 .query({ access_token: token })
-                .send({ address: [ 'user1' ] })
+                .send({ addresses: [ 'user1' ] })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(202);
                     done();
@@ -673,17 +672,9 @@ describe('Mail API', function () {
                 });
         });
 
-        it('add fails if user does not exist', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + 'someuserdoesnotexist')
-                .query({ access_token: token })
-                .end(function (err, res) {
-                    expect(res.statusCode).to.equal(404);
-                    done();
-                });
-        });
-
-        it('add/enable succeeds', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + userId)
+        it('add succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes')
+                .send({ name: MAILBOX_NAME, userId: userId })
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
@@ -691,11 +682,12 @@ describe('Mail API', function () {
                 });
         });
 
-        it('enable again succeeds if already enabled', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + userId)
+        it('cannot add again', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes')
+                .send({ name: MAILBOX_NAME, userId: userId })
                 .query({ access_token: token })
                 .end(function (err, res) {
-                    expect(res.statusCode).to.equal(201);
+                    expect(res.statusCode).to.equal(409);
                     done();
                 });
         });
@@ -710,12 +702,12 @@ describe('Mail API', function () {
         });
 
         it('get succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + userId)
+            superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + MAILBOX_NAME)
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(200);
                     expect(res.body.mailbox).to.be.an('object');
-                    expect(res.body.mailbox.name).to.equal(USERNAME);
+                    expect(res.body.mailbox.name).to.equal(MAILBOX_NAME);
                     expect(res.body.mailbox.ownerId).to.equal(userId);
                     expect(res.body.mailbox.ownerType).to.equal('user');
                     expect(res.body.mailbox.aliasTarget).to.equal(null);
@@ -731,7 +723,7 @@ describe('Mail API', function () {
                     expect(res.statusCode).to.equal(200);
                     expect(res.body.mailboxes.length).to.eql(1);
                     expect(res.body.mailboxes[0]).to.be.an('object');
-                    expect(res.body.mailboxes[0].name).to.equal(USERNAME);
+                    expect(res.body.mailboxes[0].name).to.equal(MAILBOX_NAME);
                     expect(res.body.mailboxes[0].ownerId).to.equal(userId);
                     expect(res.body.mailboxes[0].ownerType).to.equal('user');
                     expect(res.body.mailboxes[0].aliasTarget).to.equal(null);
@@ -740,21 +732,21 @@ describe('Mail API', function () {
                 });
         });
 
-        it('disable succeeds even if not exist', function (done) {
+        it('disable fails even if not exist', function (done) {
             superagent.del(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + 'someuserdoesnotexist')
                 .query({ access_token: token })
                 .end(function (err, res) {
-                    expect(res.statusCode).to.equal(201);
+                    expect(res.statusCode).to.equal(404);
                     done();
                 });
         });
 
         it('disable succeeds', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + userId)
+            superagent.del(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + MAILBOX_NAME)
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
-                    superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + userId)
+                    superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + MAILBOX_NAME)
                         .query({ access_token: token })
                         .end(function (err, res) {
                             expect(res.statusCode).to.equal(404);
@@ -790,7 +782,7 @@ describe('Mail API', function () {
         });
 
         it('set fails if aliases is missing', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + userId)
+            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + MAILBOX_NAME)
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(400);
@@ -809,7 +801,7 @@ describe('Mail API', function () {
         });
 
         it('set fails if aliases is the wrong type', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + userId)
+            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + MAILBOX_NAME)
                 .send({ aliases: 'hello, there' })
                 .query({ access_token: token })
                 .end(function (err, res) {
@@ -819,7 +811,7 @@ describe('Mail API', function () {
         });
 
         it('set fails if user is not enabled', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + userId)
+            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + MAILBOX_NAME)
                 .send({ aliases: ['hello', 'there'] })
                 .query({ access_token: token })
                 .end(function (err, res) {
@@ -828,8 +820,9 @@ describe('Mail API', function () {
                 });
         });
 
-        it('now enable the user', function (done) {
-            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes/' + userId)
+        it('now add the mailbox', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/mailboxes')
+                .send({ name: MAILBOX_NAME, userId: userId })
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
@@ -838,7 +831,7 @@ describe('Mail API', function () {
         });
 
         it('set succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + userId)
+            superagent.put(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + MAILBOX_NAME)
                 .send({ aliases: ['hello', 'there'] })
                 .query({ access_token: token })
                 .end(function (err, res) {
@@ -848,7 +841,7 @@ describe('Mail API', function () {
         });
 
         it('get succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + userId)
+            superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + MAILBOX_NAME)
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(200);
@@ -857,7 +850,27 @@ describe('Mail API', function () {
                 });
         });
 
-        it('get succeeds if mailbox does not exist', function (done) {
+        it('listing succeeds', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(200);
+                    expect(res.body.aliases.length).to.eql(2);
+                    expect(res.body.aliases[0].name).to.equal('hello');
+                    expect(res.body.aliases[0].ownerId).to.equal(userId);
+                    expect(res.body.aliases[0].ownerType).to.equal('user');
+                    expect(res.body.aliases[0].aliasTarget).to.equal(MAILBOX_NAME);
+                    expect(res.body.aliases[0].domain).to.equal(DOMAIN_0.domain);
+                    expect(res.body.aliases[1].name).to.equal('there');
+                    expect(res.body.aliases[1].ownerId).to.equal(userId);
+                    expect(res.body.aliases[1].ownerType).to.equal('user');
+                    expect(res.body.aliases[1].aliasTarget).to.equal(MAILBOX_NAME);
+                    expect(res.body.aliases[1].domain).to.equal(DOMAIN_0.domain);
+                    done();
+                });
+        });
+
+        it('get fails if mailbox does not exist', function (done) {
             superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/aliases/' + 'someuserdoesnotexist')
                 .query({ access_token: token })
                 .end(function (err, res) {
@@ -878,25 +891,6 @@ describe('Mail API', function () {
                             expect(res.statusCode).to.equal(201);
                             done();
                         });
-                },
-                function (done) {
-                    superagent.post(SERVER_URL + '/api/v1/groups')
-                        .query({ access_token: token })
-                        .send({ name: GROUP_NAME})
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(201);
-                            groupObject = result.body;
-                            done();
-                        });
-                },
-                function (done) {
-                    superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
-                        .query({ access_token: token })
-                        .send({ groupIds: [ 'admin', groupObject.id ]})
-                        .end(function (error, result) {
-                            expect(result.statusCode).to.equal(204);
-                            done();
-                        });
                 }
             ], done);
         });
@@ -934,19 +928,29 @@ describe('Mail API', function () {
                 });
         });
 
-        it('add fails with non-existing groupId', function (done) {
+        it('add fails without members array', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists')
-                .send({ groupId: 'doesnotexist' })
+                .send({ name: LIST_NAME })
                 .query({ access_token: token })
                 .end(function (err, res) {
-                    expect(res.statusCode).to.equal(404);
+                    expect(res.statusCode).to.equal(400);
+                    done();
+                });
+        });
+
+        it('cannot add reserved group', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists')
+                .send({ name: LIST_NAME, members: [ 'Admin', USERNAME ]})
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(400);
                     done();
                 });
         });
 
         it('add succeeds', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists')
-                .send({ groupId: groupObject.id })
+                .send({ name: LIST_NAME, members: [ 'admin2', USERNAME ]})
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(201);
@@ -956,7 +960,7 @@ describe('Mail API', function () {
 
         it('add twice fails', function (done) {
             superagent.post(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists')
-                .send({ groupId: groupObject.id })
+                .send({ name: LIST_NAME, members: [ 'admin2', USERNAME ] })
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(409);
@@ -974,17 +978,17 @@ describe('Mail API', function () {
         });
 
         it('get succeeds', function (done) {
-            superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists/' + groupObject.id)
+            superagent.get(SERVER_URL + `/api/v1/mail/${DOMAIN_0.domain}/lists/${LIST_NAME}`)
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(200);
                     expect(res.body.list).to.be.an('object');
-                    expect(res.body.list.name).to.equal(GROUP_NAME);
-                    expect(res.body.list.ownerId).to.equal(groupObject.id);
+                    expect(res.body.list.name).to.equal(LIST_NAME);
+                    expect(res.body.list.ownerId).to.equal('admin');
                     expect(res.body.list.ownerType).to.equal('group');
                     expect(res.body.list.aliasTarget).to.equal(null);
                     expect(res.body.list.domain).to.equal(DOMAIN_0.domain);
-                    expect(res.body.list.members).to.eql([ 'superadmin' ]);
+                    expect(res.body.list.members).to.eql([ 'admin2', 'superadmin' ]);
                     done();
                 });
         });
@@ -996,11 +1000,12 @@ describe('Mail API', function () {
                     expect(res.statusCode).to.equal(200);
                     expect(res.body.lists).to.be.an(Array);
                     expect(res.body.lists.length).to.equal(1);
-                    expect(res.body.lists[0].name).to.equal(GROUP_NAME);
-                    expect(res.body.lists[0].ownerId).to.equal(groupObject.id);
+                    expect(res.body.lists[0].name).to.equal(LIST_NAME);
+                    expect(res.body.lists[0].ownerId).to.equal('admin');
                     expect(res.body.lists[0].ownerType).to.equal('group');
                     expect(res.body.lists[0].aliasTarget).to.equal(null);
                     expect(res.body.lists[0].domain).to.equal(DOMAIN_0.domain);
+                    expect(res.body.lists[0].members).to.eql([ 'admin2', 'superadmin' ]);
                     done();
                 });
         });
@@ -1015,12 +1020,12 @@ describe('Mail API', function () {
         });
 
         it('del succeeds', function (done) {
-            superagent.del(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists/' + groupObject.id)
+            superagent.del(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists/' + LIST_NAME)
                 .query({ access_token: token })
                 .end(function (err, res) {
                     expect(res.statusCode).to.equal(204);
 
-                    superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists/' + groupObject.id)
+                    superagent.get(SERVER_URL + '/api/v1/mail/' + DOMAIN_0.domain + '/lists/' + LIST_NAME)
                         .query({ access_token: token })
                         .end(function (err, res) {
                             expect(res.statusCode).to.equal(404);

src/routes/test/user-test.js

@@ -44,7 +44,7 @@ function setup(done) {
         database._clear,
         mailer._clearMailQueue,
         domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-        mail.add.bind(null, DOMAIN_0.domain)
+        mail.addDomain.bind(null, DOMAIN_0.domain)
     ], function (error) {
         expect(error).to.not.be.ok();
 

src/server.js

@@ -214,10 +214,11 @@ function initializeExpressSync() {
     router.post('/api/v1/settings/appstore_config',    settingsScope, routes.user.requireAdmin, routes.settings.setAppstoreConfig);
 
     // email routes
-    router.get ('/api/v1/mail/:domain',       settingsScope, routes.user.requireAdmin, routes.mail.get);
-    router.post('/api/v1/mail/:domain',       settingsScope, routes.user.requireAdmin, routes.mail.update);
-    router.post('/api/v1/mail',               settingsScope, routes.user.requireAdmin, routes.mail.add);
-    router.del ('/api/v1/mail/:domain',       settingsScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.mail.del);
+    router.get ('/api/v1/mail/:domain',       settingsScope, routes.user.requireAdmin, routes.mail.getDomain);
+    router.post('/api/v1/mail/:domain',       settingsScope, routes.user.requireAdmin, routes.mail.updateDomain);
+    router.post('/api/v1/mail',               settingsScope, routes.user.requireAdmin, routes.mail.addDomain);
+    router.get ('/api/v1/mail/:domain/stats', settingsScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.mail.getDomainStats);
+    router.del ('/api/v1/mail/:domain',       settingsScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.mail.removeDomain);
     router.get ('/api/v1/mail/:domain/status',       settingsScope, routes.user.requireAdmin, routes.mail.getStatus);
     router.post('/api/v1/mail/:domain/mail_from_validation', settingsScope, routes.user.requireAdmin, routes.mail.setMailFromValidation);
     router.post('/api/v1/mail/:domain/catch_all',  settingsScope, routes.user.requireAdmin, routes.mail.setCatchAllAddress);
@@ -225,15 +226,18 @@ function initializeExpressSync() {
     router.post('/api/v1/mail/:domain/enable',        settingsScope, routes.user.requireAdmin, routes.mail.setMailEnabled);
     router.post('/api/v1/mail/:domain/send_test_mail',  settingsScope, routes.user.requireAdmin, routes.mail.sendTestMail);
     router.get ('/api/v1/mail/:domain/mailboxes',  settingsScope, routes.user.requireAdmin, routes.mail.getMailboxes);
-    router.get ('/api/v1/mail/:domain/mailboxes/:userId',  settingsScope, routes.user.requireAdmin, routes.mail.getUserMailbox);
-    router.post('/api/v1/mail/:domain/mailboxes/:userId',  settingsScope, routes.user.requireAdmin, routes.mail.enableUserMailbox);
-    router.del ('/api/v1/mail/:domain/mailboxes/:userId',  settingsScope, routes.user.requireAdmin, routes.mail.disableUserMailbox);
-    router.get ('/api/v1/mail/:domain/aliases/:userId', settingsScope, routes.user.requireAdmin, routes.mail.getAliases);
-    router.put ('/api/v1/mail/:domain/aliases/:userId', settingsScope, routes.user.requireAdmin, routes.mail.setAliases);
+    router.get ('/api/v1/mail/:domain/mailboxes/:name',  settingsScope, routes.user.requireAdmin, routes.mail.getMailbox);
+    router.post('/api/v1/mail/:domain/mailboxes',  settingsScope, routes.user.requireAdmin, routes.mail.addMailbox);
+    router.post('/api/v1/mail/:domain/mailboxes/:name',  settingsScope, routes.user.requireAdmin, routes.mail.updateMailbox);
+    router.del ('/api/v1/mail/:domain/mailboxes/:name',  settingsScope, routes.user.requireAdmin, routes.mail.removeMailbox);
+    router.get ('/api/v1/mail/:domain/aliases', settingsScope, routes.user.requireAdmin, routes.mail.listAliases);
+    router.get ('/api/v1/mail/:domain/aliases/:name', settingsScope, routes.user.requireAdmin, routes.mail.getAliases);
+    router.put ('/api/v1/mail/:domain/aliases/:name', settingsScope, routes.user.requireAdmin, routes.mail.setAliases);
     router.get ('/api/v1/mail/:domain/lists', settingsScope, routes.user.requireAdmin, routes.mail.getLists);
     router.post('/api/v1/mail/:domain/lists', settingsScope, routes.user.requireAdmin, routes.mail.addList);
-    router.get ('/api/v1/mail/:domain/lists/:groupId', settingsScope, routes.user.requireAdmin, routes.mail.getList);
-    router.del ('/api/v1/mail/:domain/lists/:groupId', settingsScope, routes.user.requireAdmin, routes.mail.removeList);
+    router.get ('/api/v1/mail/:domain/lists/:name', settingsScope, routes.user.requireAdmin, routes.mail.getList);
+    router.post('/api/v1/mail/:domain/lists/:name', settingsScope, routes.user.requireAdmin, routes.mail.updateList);
+    router.del ('/api/v1/mail/:domain/lists/:name', settingsScope, routes.user.requireAdmin, routes.mail.removeList);
 
     // feedback
     router.post('/api/v1/feedback', usersScope, routes.cloudron.feedback);

src/setup.js

@@ -197,7 +197,7 @@ function dnsSetup(adminFqdn, domain, zoneName, provider, dnsConfig, tlsConfig, c
 
         async.series([
             domains.add.bind(null, domain, zoneName, provider, dnsConfig, null /* cert */, tlsConfig),
-            mail.add.bind(null, domain)
+            mail.addDomain.bind(null, domain)
         ], done);
     });
 }

src/storage/s3.js

@@ -75,7 +75,8 @@ function getCaasConfig(apiConfig, callback) {
                 customBackoff: () => 20000 // constant backoff - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#retryDelayOptions-property
             },
             httpOptions: {
-                connectTimeout: 10000 // https://github.com/aws/aws-sdk-js/pull/1446
+                connectTimeout: 10000, // https://github.com/aws/aws-sdk-js/pull/1446
+                timeout: 300 * 1000 // https://github.com/aws/aws-sdk-js/issues/1704 (allow 5MB chunk upload to take upto 5 minutes)
             }
         };
 
@@ -107,7 +108,8 @@ function getS3Config(apiConfig, callback) {
             customBackoff: () => 20000 // constant backoff - https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html#retryDelayOptions-property
         },
         httpOptions: {
-            connectTimeout: 10000 // https://github.com/aws/aws-sdk-js/pull/1446
+            connectTimeout: 10000, // https://github.com/aws/aws-sdk-js/pull/1446
+            timeout: 300 * 1000 // https://github.com/aws/aws-sdk-js/issues/1704 (allow 5MB chunk upload to take upto 5 minutes)
         }
     };
 
@@ -139,12 +141,14 @@ function upload(apiConfig, backupFilePath, sourceStream, callback) {
 
         // s3.upload automatically does a multi-part upload. we set queueSize to 1 to reduce memory usage
         // uploader will buffer at most queueSize * partSize bytes into memory at any given time.
-        s3.upload(params, { partSize: 10 * 1024 * 1024, queueSize: 1 }, function (error) {
+        s3.upload(params, { partSize: 10 * 1024 * 1024, queueSize: 1 }, function (error, data) {
             if (error) {
-                debug('[%s] upload: s3 upload error.', backupFilePath, error);
+                debug('Error uploading [%s]: s3 upload error.', backupFilePath, error);
                 return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error uploading ${backupFilePath}. Message: ${error.message} HTTP Code: ${error.code}`));
             }
 
+            debug(`Uploaded ${backupFilePath}: ${JSON.stringify(data)}`);
+
             callback(null);
         });
     });
@@ -294,11 +298,10 @@ function copy(apiConfig, oldFilePath, newFilePath) {
         var relativePath = path.relative(oldFilePath, content.Key);
 
         function done(error) {
+            if (error) debug(`copy: s3 copy error when copying ${content.Key}: ${error}`);
+
             if (error && error.code === 'NoSuchKey') return iteratorCallback(new BackupsError(BackupsError.NOT_FOUND, `Old backup not found: ${content.Key}`));
-            if (error) {
-                debug('copy: s3 copy error when copying %s %s', content.Key, error);
-                return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.message} ${error.code}`));
-            }
+            if (error) return iteratorCallback(new BackupsError(BackupsError.EXTERNAL_ERROR, `Error copying ${content.Key} : ${error.code} ${error}`));
 
             iteratorCallback(null);
         }
@@ -318,7 +321,9 @@ function copy(apiConfig, oldFilePath, newFilePath) {
             copyParams.CopySource = encodeCopySource(apiConfig.bucket, content.Key);
             s3.copyObject(copyParams, done).on('retry', function (response) {
                 ++retryCount;
-                events.emit('progress', `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Status code: ${response.httpResponse.statusCode}`);
+                events.emit('progress', `Retrying (${response.retryCount+1}) copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
+                // on DO, we get a random 408. these are not retried by the SDK
+                if (response.error) response.error.retryable = true; // https://github.com/aws/aws-sdk-js/issues/412
             });
 
             return;
@@ -372,7 +377,7 @@ function copy(apiConfig, oldFilePath, newFilePath) {
                     s3.completeMultipartUpload(params, done);
                 }).on('retry', function (response) {
                     ++retryCount;
-                    events.emit('progress', `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Status code: ${response.httpResponse.statusCode}`);
+                    events.emit('progress', `Retrying (${response.retryCount+1}) multipart copy of ${relativePath || oldFilePath}. Error: ${response.error} ${response.httpResponse.statusCode}`);
                 });
             }
 
@@ -386,12 +391,12 @@ function copy(apiConfig, oldFilePath, newFilePath) {
         total += objects.length;
 
         if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
-        events.emit('progress', `${retryCount} errors so far. concurrency set to ${concurrency}`);
+        events.emit('progress', `Copying ${total-objects.length}-${total}. ${retryCount} errors so far. concurrency set to ${concurrency}`);
         retryCount = 0;
 
         async.eachLimit(objects, concurrency, copyFile.bind(null, s3), done);
     }, function (error) {
-        events.emit('progress', `Copied ${total} files`);
+        events.emit('progress', `Copied ${total} files with error: ${error}`);
 
         events.emit('done', error);
     });

src/syncer.js

@@ -104,14 +104,14 @@ function sync(dir, taskProcessor, concurrency, callback) {
                 if (entryStat.isDirectory()) {
                     traverse(entryPath);
                 } else {
-                    addQueue.push({ operation: 'add', path: entryPath, reason: 'new' });
+                    addQueue.push({ operation: 'add', path: entryPath, reason: 'new', position: addQueue.length });
                 }
             } else if (ISDIR(cacheStat.mode) && entryStat.isDirectory()) { // dir names match
                 ++curCacheIndex;
                 traverse(entryPath);
             } else if (ISFILE(cacheStat.mode) && entryStat.isFile()) { // file names match
                 if (entryStat.mtime.getTime() !== cacheStat.mtime || entryStat.size != cacheStat.size || entryStat.inode !== cacheStat.inode) { // file changed
-                    addQueue.push({ operation: 'add', path: entryPath, reason: 'changed' });
+                    addQueue.push({ operation: 'add', path: entryPath, reason: 'changed', position: addQueue.length });
                 }
                 ++curCacheIndex;
             } else if (entryStat.isDirectory()) { // was a file, now a directory
@@ -121,7 +121,7 @@ function sync(dir, taskProcessor, concurrency, callback) {
             } else { // was a dir, now a file
                 delQueue.push({ operation: 'removedir', path: cachePath, reason: 'wasdir' });
                 while (curCacheIndex !== cache.length && cache[curCacheIndex].path.startsWith(cachePath)) ++curCacheIndex;
-                addQueue.push({ operation: 'add', path: entryPath, reason: 'wasdir' });
+                addQueue.push({ operation: 'add', path: entryPath, reason: 'wasdir', position: addQueue.length });
             }
         }
     }

src/test/database-test.js

@@ -681,7 +681,7 @@ describe('database', function () {
             tokendb.add(TOKEN_0.accessToken, TOKEN_0.identifier, TOKEN_0.clientId, TOKEN_0.expires, TOKEN_0.scope, function (error) {
                 expect(error).to.be(null);
 
-                tokendb.delByClientId(TOKEN_0.clientId, function (error, result) {
+                tokendb.delByClientId(TOKEN_0.clientId, function (error) {
                     expect(error).to.not.be.ok();
 
                     tokendb.get(TOKEN_0.accessToken, function (error, result) {
@@ -1016,7 +1016,7 @@ describe('database', function () {
         });
 
         it('getAddonConfigByName of unknown value succeeds', function (done) {
-            appdb.getAddonConfigByName(APP_1.id, 'addonid1', 'NOPE', function (error, value) {
+            appdb.getAddonConfigByName(APP_1.id, 'addonid1', 'NOPE', function (error) {
                 expect(error.reason).to.be(DatabaseError.NOT_FOUND);
                 done();
             });
@@ -1412,17 +1412,12 @@ describe('database', function () {
             }, function (error) {
                 expect(error).to.be(null);
 
-                var actions = [ 'anotherpersistent.event', 'persistent.event' ];
-
-                eventlogdb.delByCreationTime(new Date(), actions, function (error) {
+                eventlogdb.delByCreationTime(new Date(), function (error) {
                     expect(error).to.be(null);
 
                     eventlogdb.getAllPaged([], null, 1, 100, function (error, results) {
                         expect(error).to.be(null);
-                        expect(results.length).to.be(2);
-                        results = results.sort(function (x, y) { return x.action > y.action; }); // because equal timestamp gives random ordering
-                        expect(results[1].action).to.be.eql('persistent.event');
-                        expect(results[0].action).to.be.eql('anotherpersistent.event');
+                        expect(results.length).to.be(0);
 
                         done();
                     });
@@ -1448,7 +1443,7 @@ describe('database', function () {
         var GROUP_ID_1 = 'foundersid';
 
         it('can create a group', function (done) {
-            groupdb.add(GROUP_ID_1, 'founders', function (error, result) {
+            groupdb.add(GROUP_ID_1, 'founders', function (error) {
                 expect(error).to.be(null);
                 done();
             });
@@ -1601,21 +1596,21 @@ describe('database', function () {
         });
 
         it('add user mailbox succeeds', function (done) {
-            mailboxdb.add('girish', DOMAIN_0.domain, 'uid-0', mailboxdb.TYPE_USER, function (error, mailbox) {
+            mailboxdb.addMailbox('girish', DOMAIN_0.domain, 'uid-0', mailboxdb.OWNER_TYPE_USER, function (error) {
                 expect(error).to.be(null);
                 done();
             });
         });
 
         it('cannot add dup entry', function (done) {
-            mailboxdb.add('girish', DOMAIN_0.domain, 'uid-1', mailboxdb.TYPE_APP, function (error, mailbox) {
+            mailboxdb.addMailbox('girish', DOMAIN_0.domain, 'uid-1', mailboxdb.OWNER_TYPE_APP, function (error) {
                 expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
                 done();
             });
         });
 
         it('add app mailbox succeeds', function (done) {
-            mailboxdb.add('support', DOMAIN_0.domain, 'osticket', mailboxdb.TYPE_APP, function (error, mailbox) {
+            mailboxdb.addMailbox('support', DOMAIN_0.domain, 'osticket', mailboxdb.OWNER_TYPE_APP, function (error) {
                 expect(error).to.be(null);
                 done();
             });
@@ -1638,7 +1633,7 @@ describe('database', function () {
                 expect(error).to.be(null);
                 expect(mailboxes.length).to.be(2);
                 expect(mailboxes[0].name).to.be('girish');
-                expect(mailboxes[0].ownerType).to.be(mailboxdb.TYPE_USER);
+                expect(mailboxes[0].ownerType).to.be(mailboxdb.OWNER_TYPE_USER);
                 expect(mailboxes[1].name).to.be('support');
 
                 done();
@@ -1740,7 +1735,7 @@ describe('database', function () {
             mailboxdb.delByOwnerId('osticket', function (error) {
                 expect(error).to.be(null);
 
-                mailboxdb.getByOwnerId('osticket', function (error, results) {
+                mailboxdb.getByOwnerId('osticket', function (error) {
                     expect(error).to.be.ok();
                     expect(error.reason).to.be(DatabaseError.NOT_FOUND);
                     done();

src/test/digest-test.js

@@ -72,7 +72,7 @@ describe('digest', function () {
             database._clear,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             function (callback) {
                 userdb.getByUsername(USER_0.username, function (error, result) {

src/test/ldap-test.js

@@ -103,7 +103,7 @@ function setup(done) {
         appdb.update.bind(null, APP_0.id, { containerId: APP_0.containerId }),
         appdb.setAddonConfig.bind(null, APP_0.id, 'sendmail', [{ name: 'MAIL_SMTP_PASSWORD', value : 'sendmailpassword' }]),
         appdb.setAddonConfig.bind(null, APP_0.id, 'recvmail', [{ name: 'MAIL_IMAP_PASSWORD', value : 'recvmailpassword' }]),
-        mailboxdb.add.bind(null, APP_0.location + '.app', APP_0.domain, APP_0.id, mailboxdb.TYPE_APP),
+        mailboxdb.addMailbox.bind(null, APP_0.location + '.app', APP_0.domain, APP_0.id, mailboxdb.OWNER_TYPE_APP),
 
         function (callback) {
             user.createOwner(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE, function (error, result) {
@@ -739,7 +739,7 @@ describe('Ldap', function () {
 
     describe('search mailbox', function () {
         before(function (done) {
-            mailboxdb.add(USER_0.username.toLowerCase(), DOMAIN_0.domain, USER_0.id, mailboxdb.TYPE_USER, done);
+            mailboxdb.addMailbox(USER_0.username.toLowerCase(), DOMAIN_0.domain, USER_0.id, mailboxdb.OWNER_TYPE_USER, done);
         });
 
         it('get specific mailbox by email', function (done) {
@@ -824,14 +824,14 @@ describe('Ldap', function () {
 
     describe('search mailing list', function () {
         before(function (done) {
-            mailboxdb.add(GROUP_NAME, DOMAIN_0.domain, GROUP_ID, mailboxdb.TYPE_GROUP, done);
+            mailboxdb.addGroup('devs', DOMAIN_0.domain, [ USER_0.username.toLowerCase(), USER_1.username.toLowerCase() ], done);
         });
 
         it('get specific list', function (done) {
-            ldapSearch('cn=developers@example.com,ou=mailinglists,dc=cloudron', 'objectclass=mailGroup', function (error, entries) {
+            ldapSearch('cn=devs@example.com,ou=mailinglists,dc=cloudron', 'objectclass=mailGroup', function (error, entries) {
                 if (error) return done(error);
                 expect(entries.length).to.equal(1);
-                expect(entries[0].cn).to.equal('developers@example.com');
+                expect(entries[0].cn).to.equal('devs@example.com');
                 expect(entries[0].mgrpRFC822MailMember).to.eql([ USER_0.username.toLowerCase() + '@example.com', USER_1.username.toLowerCase() + '@example.com' ]);
                 done();
             });

src/test/mail-test.js

@@ -31,7 +31,7 @@ function setup(done) {
         database.initialize,
         database._clear,
         domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-        mail.add.bind(null, DOMAIN_0.domain)
+        mail.addDomain.bind(null, DOMAIN_0.domain)
     ], done);
 }
 
@@ -48,7 +48,7 @@ describe('Mail', function () {
 
     describe('values', function () {
         it('can get default', function (done) {
-            mail.get(DOMAIN_0.domain, function (error, mailConfig) {
+            mail.getDomain(DOMAIN_0.domain, function (error, mailConfig) {
                 expect(error).to.be(null);
                 expect(mailConfig.enabled).to.be(false);
                 expect(mailConfig.mailFromValidation).to.be(true);
@@ -62,7 +62,7 @@ describe('Mail', function () {
             mail.setMailFromValidation(DOMAIN_0.domain, false, function (error) {
                 expect(error).to.be(null);
 
-                mail.get(DOMAIN_0.domain, function (error, mailConfig) {
+                mail.getDomain(DOMAIN_0.domain, function (error, mailConfig) {
                     expect(error).to.be(null);
                     expect(mailConfig.mailFromValidation).to.be(false);
 
@@ -75,7 +75,7 @@ describe('Mail', function () {
             mail.setCatchAllAddress(DOMAIN_0.domain, [ 'user1', 'user2' ], function (error) {
                 expect(error).to.be(null);
 
-                mail.get(DOMAIN_0.domain, function (error, mailConfig) {
+                mail.getDomain(DOMAIN_0.domain, function (error, mailConfig) {
                     expect(error).to.be(null);
                     expect(mailConfig.catchAll).to.eql([ 'user1', 'user2' ]);
                     done();
@@ -89,7 +89,7 @@ describe('Mail', function () {
             maildb.update(DOMAIN_0.domain, { relay: relay }, function (error) { // skip the mail server verify()
                 expect(error).to.be(null);
 
-                mail.get(DOMAIN_0.domain, function (error, mailConfig) {
+                mail.getDomain(DOMAIN_0.domain, function (error, mailConfig) {
                     expect(error).to.be(null);
                     expect(mailConfig.relay).to.eql(relay);
                     done();
@@ -101,7 +101,7 @@ describe('Mail', function () {
             mail.setMailEnabled(DOMAIN_0.domain, true, function (error) {
                 expect(error).to.be(null);
 
-                mail.get(DOMAIN_0.domain, function (error, mailConfig) {
+                mail.getDomain(DOMAIN_0.domain, function (error, mailConfig) {
                     expect(error).to.be(null);
                     expect(mailConfig.enabled).to.be(true);
                     done();

src/test/syncer-test.js

@@ -53,9 +53,9 @@ describe('Syncer', function () {
             expect(error).to.not.be.ok();
 
             expect(gTasks).to.eql([
-                { operation: 'add', path: 'src/index.js', reason: 'new' },
-                { operation: 'add', path: 'test/test.js', reason: 'new' },
-                { operation: 'add', path: 'walrus', reason: 'new' }
+                { operation: 'add', path: 'src/index.js', reason: 'new', position: 0 },
+                { operation: 'add', path: 'test/test.js', reason: 'new', position: 1 },
+                { operation: 'add', path: 'walrus', reason: 'new', position: 2 }
             ]);
             done();
         });
@@ -70,7 +70,7 @@ describe('Syncer', function () {
             expect(error).to.not.be.ok();
 
             expect(gTasks).to.eql([
-                { operation: 'add', path: 'a/b/c/d/e', reason: 'new' }
+                { operation: 'add', path: 'a/b/c/d/e', reason: 'new', position: 0 }
             ]);
             done();
         });
@@ -85,7 +85,7 @@ describe('Syncer', function () {
             expect(error).to.not.be.ok();
 
             expect(gTasks).to.eql([
-                { operation: 'add', path: 'readme', reason: 'new' }
+                { operation: 'add', path: 'readme', reason: 'new', position: 0 }
             ]);
             done();
         });
@@ -107,8 +107,8 @@ describe('Syncer', function () {
                 expect(error).to.not.be.ok();
 
                 expect(gTasks).to.eql([
-                    { operation: 'add', path: 'src/index.js', reason: 'changed' },
-                    { operation: 'add', path: 'test/test.js', reason: 'changed' }
+                    { operation: 'add', path: 'src/index.js', reason: 'changed', position: 0 },
+                    { operation: 'add', path: 'test/test.js', reason: 'changed', position: 1 }
                 ]);
 
                 done();
@@ -209,7 +209,7 @@ describe('Syncer', function () {
 
                 expect(gTasks).to.eql([
                     { operation: 'removedir', path: 'a/b', reason: 'missing' },
-                    { operation: 'add', path: 'a/f', reason: 'new' }
+                    { operation: 'add', path: 'a/f', reason: 'new', position: 0 }
                 ]);
 
                 done();
@@ -234,7 +234,7 @@ describe('Syncer', function () {
 
                 expect(gTasks).to.eql([
                     { operation: 'remove', path: 'data/test/test.js', reason: 'wasfile' },
-                    { operation: 'add', path: 'data/test/test.js/trick', reason: 'new' }
+                    { operation: 'add', path: 'data/test/test.js/trick', reason: 'new', position: 0 }
                 ]);
 
                 done();
@@ -259,7 +259,7 @@ describe('Syncer', function () {
 
                 expect(gTasks).to.eql([
                     { operation: 'removedir', path: 'test', reason: 'wasdir' },
-                    { operation: 'add', path: 'test', reason: 'wasdir' }
+                    { operation: 'add', path: 'test', reason: 'wasdir', position: 0 }
                 ]);
 
                 done();
@@ -312,9 +312,9 @@ describe('Syncer', function () {
                     { operation: 'removedir', path: 'j/l', reason: 'missing' },
                     { operation: 'removedir', path: 'j/m', reason: 'missing' },
 
-                    { operation: 'add', path: 'a/file', reason: 'new' },
-                    { operation: 'add', path: 'b', reason: 'changed' },
-                    { operation: 'add', path: 'j/k/file', reason: 'new' },
+                    { operation: 'add', path: 'a/file', reason: 'new', position: 0 },
+                    { operation: 'add', path: 'b', reason: 'changed', position: 1 },
+                    { operation: 'add', path: 'j/k/file', reason: 'new', position: 2 },
                 ]);
 
                 gTasks = [ ];

src/test/updatechecker-test.js

@@ -76,7 +76,7 @@ describe('updatechecker - box - manual (email)', function () {
             database._clear,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settings.setBoxAutoupdatePattern.bind(null, constants.AUTOUPDATE_PATTERN_NEVER),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' })),
@@ -178,7 +178,7 @@ describe('updatechecker - box - automatic (no email)', function () {
             database.initialize,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             mailer._clearMailQueue,
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' }))
@@ -222,7 +222,7 @@ describe('updatechecker - box - automatic free (email)', function () {
             database.initialize,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             mailer._clearMailQueue,
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' }))
@@ -292,7 +292,7 @@ describe('updatechecker - app - manual (email)', function () {
             database._clear,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             mailer._clearMailQueue,
             appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.portBindings, APP_0),
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
@@ -408,7 +408,7 @@ describe('updatechecker - app - automatic (no email)', function () {
             database._clear,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             mailer._clearMailQueue,
             appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.portBindings, APP_0),
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
@@ -474,7 +474,7 @@ describe('updatechecker - app - automatic free (email)', function () {
             database._clear,
             settings.initialize,
             domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-            mail.add.bind(null, DOMAIN_0.domain),
+            mail.addDomain.bind(null, DOMAIN_0.domain),
             mailer._clearMailQueue,
             appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.domain, APP_0.portBindings, APP_0),
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),

src/test/user-test.js

@@ -78,7 +78,7 @@ function setup(done) {
         database.initialize,
         database._clear,
         domains.add.bind(null, DOMAIN_0.domain, DOMAIN_0.zoneName, DOMAIN_0.provider, DOMAIN_0.config, DOMAIN_0.fallbackCertificate, DOMAIN_0.tlsConfig),
-        mail.add.bind(null, DOMAIN_0.domain),
+        mail.addDomain.bind(null, DOMAIN_0.domain),
         mailer._clearMailQueue
     ], done);
 }

src/user.js

@@ -36,7 +36,6 @@ var assert = require('assert'),
     groups = require('./groups.js'),
     GroupError = groups.GroupError,
     hat = require('hat'),
-    mailboxdb = require('./mailboxdb.js'),
     mailer = require('./mailer.js'),
     safe = require('safetydance'),
     tokendb = require('./tokendb.js'),
@@ -268,19 +267,15 @@ function removeUser(userId, auditSource, callback) {
 
         if (config.isDemo() && user.username === constants.DEMO_USERNAME) return callback(new UserError(UserError.BAD_FIELD, 'Not allowed in demo mode'));
 
-        mailboxdb.delByOwnerId(userId, function (error) {
+        userdb.del(userId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
             if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
-            userdb.del(userId, function (error) {
-                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
-                if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
+            eventlog.add(eventlog.ACTION_USER_REMOVE, auditSource, { userId: userId, user: removePrivateFields(user) });
 
-                eventlog.add(eventlog.ACTION_USER_REMOVE, auditSource, { userId: userId, user: removePrivateFields(user) });
+            callback();
 
-                callback();
-
-                mailer.userRemoved(user);
-            });
+            mailer.userRemoved(user);
         });
     });
 }
@@ -372,7 +367,7 @@ function updateUser(userId, data, auditSource, callback) {
         if (error) return callback(error);
     }
 
-    userdb.get(userId, function (error, user) {
+    userdb.get(userId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));