Keep restarting mysql until it succeeds

MySQL restarts randomly fail on our CI systems. This is easily
reproducible:

root@smartserver:~# cp /tmp/mysql.cnf . && systemctl restart mysql && echo "Yes"
Yes
root@smartserver:~# cp /tmp/mysql.cnf . && systemctl restart mysql && echo "Yes"
Yes
root@smartserver:~# cp /tmp/mysql.cnf . && systemctl restart mysql && echo "Yes"
Job for mysql.service failed. See "systemctl status mysql.service" and "journalctl -xe" for details.

There also seems some apparmor issue:
[ 7389.111704] audit: type=1400 audit(1509404778.110:829): apparmor="DENIED" operation="open" profile="/usr/sbin/mysqld" name="/sys/devices/system/node/" pid=15618 comm="mysqld" requested_mask="r" denied_mask="r" fsuid=112 ouid=0

The apparmor issue is reported in https://bugs.launchpad.net/ubuntu/+source/mysql-5.7/+bug/1610765,
https://bugs.launchpad.net/ubuntu/+source/mysql-5.7/+bug/1658233 and
https://bugs.launchpad.net/ubuntu/+source/apparmor/+bug/1658239

CHANGES

@@ -1079,3 +1079,18 @@
 * Fix crash in carbon which made graphs disappear on some Cloudrons
 * Fix issue where OAuth SSO did not work when alternate domain was used
 * Changelog is now rendered in markdown format
+
+[1.7.5]
+* Expose a TLS relay port from mail container for Go applications
+
+[1.7.6]
+* Port bindings cannot be configured in update route anymore
+* Implement LDAP group compare
+* Pre-releases are now offered by appstore and not handled in box code anymore
+* LDAP pagination support. This will fix the warnings in NextCloud and Rocket.Chat
+* Check if directories can be created in the backup directory
+* Do not set the HTTPS agent when using HTTP with minio backup backend
+* Fix regression where a new domain config could not be set in the UI
+* New mail container release that fixes email sending with SOGo
+* Show 404 page for unknown domains
+

scripts/cloudron-setup

@@ -45,6 +45,7 @@ fi
 initBaseImage="true"
 # provisioning data
 domain=""
+adminLocation="my"
 zoneName=""
 provider=""
 encryptionKey=""
@@ -63,13 +64,14 @@ baseDataDir=""
 # TODO this is still there for the restore case, see other occasions below
 versionsUrl="https://s3.amazonaws.com/prod-cloudron-releases/versions.json"
 
-args=$(getopt -o "" -l "domain:,help,skip-baseimage-init,data:,data-dir:,provider:,encryption-key:,restore-url:,tls-provider:,version:,dns-provider:,env:,prerelease,skip-reboot,source-url:" -n "$0" -- "$@")
+args=$(getopt -o "" -l "domain:,help,skip-baseimage-init,data:,data-dir:,provider:,encryption-key:,restore-url:,tls-provider:,version:,dns-provider:,env:,admin-location:,prerelease,skip-reboot,source-url:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
     --domain) domain="$2"; shift 2;;
-    --help) echo "See https://cloudron.io/references/selfhosting.html on how to install Cloudron"; exit 0;;
+    --admin-location) adminLocation="$2"; shift 2;;
+    --help) echo "See https://cloudron.io/documentation/installation/ on how to install Cloudron"; exit 0;;
     --provider) provider="$2"; shift 2;;
     --encryption-key) encryptionKey="$2"; shift 2;;
     --restore-url) restoreUrl="$2"; shift 2;;
@@ -105,12 +107,12 @@ done
 # validate arguments in the absence of data
 if [[ -z "${dataJson}" ]]; then
     if [[ -z "${provider}" ]]; then
-        echo "--provider is required (azure, cloudscale, digitalocean, ec2, exoscale, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic)"
+        echo "--provider is required (azure, cloudscale.ch, digitalocean, ec2, exoscale, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic)"
         exit 1
     elif [[  \
                 "${provider}" != "ami" && \
                 "${provider}" != "azure" && \
-                "${provider}" != "cloudscale" && \
+                "${provider}" != "cloudscale.ch" && \
                 "${provider}" != "digitalocean" && \
                 "${provider}" != "ec2" && \
                 "${provider}" != "exoscale" && \
@@ -123,7 +125,7 @@ if [[ -z "${dataJson}" ]]; then
                 "${provider}" != "vultr" && \
                 "${provider}" != "generic" \
             ]]; then
-        echo "--provider must be one of: azure, cloudscale, digitalocean, ec2, exoscale, gce, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic"
+        echo "--provider must be one of: azure, cloudscale.ch, digitalocean, ec2, exoscale, gce, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic"
         exit 1
     fi
 
@@ -197,6 +199,7 @@ if [[ -z "${dataJson}" ]]; then
     {
         "boxVersionsUrl": "${versionsUrl}",
         "fqdn": "${domain}",
+        "adminLocation": "${adminLocation}",
         "zoneName": "${zoneName}",
         "provider": "${provider}",
         "apiServerOrigin": "${apiServerOrigin}",
@@ -214,9 +217,6 @@ if [[ -z "${dataJson}" ]]; then
             "format": "tgz",
             "retentionSecs": 172800
         },
-        "updateConfig": {
-            "prerelease": ${prerelease}
-        },
         "version": "${version}"
     }
 EOF
@@ -226,6 +226,7 @@ EOF
     {
         "boxVersionsUrl": "${versionsUrl}",
         "fqdn": "${domain}",
+        "adminLocation": "${adminLocation}",
         "zoneName": "${zoneName}",
         "provider": "${provider}",
         "apiServerOrigin": "${apiServerOrigin}",

setup/argparser.sh

@@ -6,6 +6,7 @@ json="${source_dir}/../node_modules/.bin/json"
 # IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
 arg_api_server_origin=""
 arg_fqdn=""
+arg_admin_location=""
 arg_zone_name=""
 arg_is_custom_domain="false"
 arg_restore_key=""
@@ -20,7 +21,6 @@ arg_version=""
 arg_web_server_origin=""
 arg_backup_config=""
 arg_dns_config=""
-arg_update_config=""
 arg_provider=""
 arg_app_bundle=""
 arg_is_demo="false"
@@ -46,13 +46,16 @@ while true; do
         arg_is_custom_domain=$(echo "$2" | $json isCustomDomain)
         [[ "${arg_is_custom_domain}" == "" ]] && arg_is_custom_domain="true"
 
+        arg_admin_location=$(echo "$2" | $json adminLocation)
+        [[ "${arg_admin_location}" == "" ]] && arg_admin_location="my"
+
         # only update/restore have this valid (but not migrate)
         arg_api_server_origin=$(echo "$2" | $json apiServerOrigin)
         [[ "${arg_api_server_origin}" == "" ]] && arg_api_server_origin="https://api.cloudron.io"
         arg_web_server_origin=$(echo "$2" | $json webServerOrigin)
         [[ "${arg_web_server_origin}" == "" ]] && arg_web_server_origin="https://cloudron.io"
 
-        # TODO check if an where this is used
+        # TODO check if and where this is used
         arg_version=$(echo "$2" | $json version)
 
         # read possibly empty parameters here
@@ -86,9 +89,6 @@ while true; do
         arg_dns_config=$(echo "$2" | $json dnsConfig)
         [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
 
-        arg_update_config=$(echo "$2" | $json updateConfig)
-        [[ "${arg_update_config}" == "null" ]] && arg_update_config=""
-
         shift 2
         ;;
     --) break;;

setup/splashpage.sh

@@ -7,7 +7,6 @@ readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly box_src_dir="$(realpath ${script_dir}/..)"
 readonly PLATFORM_DATA_DIR="/home/yellowtent/platformdata"
-readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
 echo "Setting up nginx update page"
 
@@ -19,7 +18,7 @@ fi
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
 # keep this is sync with config.js appFqdn()
-admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${arg_admin_location}.${arg_fqdn}" ||  echo "${arg_admin_location}-${arg_fqdn}")
 admin_origin="https://${admin_fqdn}"
 
 # copy the website

setup/start.sh

@@ -186,7 +186,11 @@ if [[ ! -f /etc/mysql/mysql.cnf ]] || ! diff -q "${script_dir}/start/mysql.cnf"
         echo "Waiting for mysql jobs..."
         sleep 1
     done
-    systemctl restart mysql
+    while true; do
+        if systemctl restart mysql; then break; fi
+        echo "Restarting MySql again after sometime since this fails randomly"
+        sleep 1
+    done
 else
     systemctl start mysql
 fi
@@ -238,6 +242,24 @@ cd "${BOX_SRC_DIR}"
 BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
 EOF
 
+echo "==> Adding automated configs"
+mysql -u root -p${mysql_root_password} -e "REPLACE INTO settings (name, value) VALUES (\"domain\", '{ \"fqdn\": \"$arg_fqdn\", \"zoneName\": \"$arg_zone_name\", \"adminLocation\": \"$arg_admin_location\" }')" box
+
+if [[ ! -z "${arg_backup_config}" ]]; then
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
+fi
+
+if [[ ! -z "${arg_dns_config}" ]]; then
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
+fi
+
+if [[ ! -z "${arg_tls_config}" ]]; then
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
+fi
+
 echo "==> Creating cloudron.conf"
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
@@ -246,6 +268,7 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "apiServerOrigin": "${arg_api_server_origin}",
     "webServerOrigin": "${arg_web_server_origin}",
     "fqdn": "${arg_fqdn}",
+    "adminLocation": "${arg_admin_location}",
     "zoneName": "${arg_zone_name}",
     "isCustomDomain": ${arg_is_custom_domain},
     "provider": "${arg_provider}",
@@ -292,27 +315,6 @@ chown "${USER}:${USER}" "${BOX_DATA_DIR}"
 find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" -exec chown -R "${USER}:${USER}" {} \;
 chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
 
-echo "==> Adding automated configs"
-if [[ ! -z "${arg_backup_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
-fi
-
-if [[ ! -z "${arg_dns_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
-fi
-
-if [[ ! -z "${arg_update_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"update_config\", '$arg_update_config')" box
-fi
-
-if [[ ! -z "${arg_tls_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
-fi
-
 set_progress "60" "Starting Cloudron"
 systemctl start cloudron.target
 

setup/start/nginx/appconfig.ejs

@@ -4,6 +4,35 @@ map $http_upgrade $connection_upgrade {
     '' close;
 }
 
+# http server
+server {
+    listen       80;
+<% if (hasIPv6) { -%>
+    listen       [::]:80;
+<% } -%>
+
+<% if (vhost) { -%>
+    server_name  <%= vhost %>;
+<% } else { -%>
+    # IP based access from collectd or initial cloudron setup. TODO: match the IPv6 address
+    server_name  "~^\d+\.\d+\.\d+\.\d+$";
+
+    # collectd
+    location /nginx_status {
+        stub_status on;
+        access_log off;
+        allow 127.0.0.1;
+        deny all;
+    }
+<% } -%>
+
+    location / {
+        # redirect everything to HTTPS
+        return 301 https://$host$request_uri;
+    }
+}
+
+# https server
 server {
 <% if (vhost) { -%>
     server_name  <%= vhost %>;

setup/start/nginx/nginx.conf

@@ -36,18 +36,12 @@ http {
     # zones for rate limiting
     limit_req_zone $binary_remote_addr zone=admin_login:10m rate=10r/s; # 10 request a second
 
-    # HTTP server
-    server {
-        listen      80;
-        listen      [::]:80;
 
-        # collectd
-        location /nginx_status {
-            stub_status on;
-            access_log off;
-            allow 127.0.0.1;
-            deny all;
-        }
+    # default http server that returns 404 for any domain we are not listening on
+    server {
+        listen      80 default_server;
+        listen      [::]:80 default_server;
+        server_name does_not_match_anything;
 
         # acme challenges
         location /.well-known/acme-challenge/ {
@@ -56,8 +50,7 @@ http {
         }
 
         location / {
-            # redirect everything to HTTPS
-            return 301 https://$host$request_uri;
+            return 404;
         }
     }
 

src/addons.js

@@ -363,6 +363,7 @@ function setupSendMail(app, options, callback) {
         var env = [
             { name: 'MAIL_SMTP_SERVER', value: 'mail' },
             { name: 'MAIL_SMTP_PORT', value: '2525' },
+            { name: 'MAIL_SMTPS_PORT', value: '4650' },
             { name: 'MAIL_SMTP_USERNAME', value: mailbox.name },
             { name: 'MAIL_SMTP_PASSWORD', value: password },
             { name: 'MAIL_FROM', value: mailbox.name + '@' + config.fqdn() },

src/appdb.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     update: update,
     getAll: getAll,
     getPortBindings: getPortBindings,
+    delPortBinding: delPortBinding,
 
     setAddonConfig: setAddonConfig,
     getAddonConfig: getAddonConfig,
@@ -252,6 +253,18 @@ function getPortBindings(id, callback) {
     });
 }
 
+function delPortBinding(hostPort, callback) {
+    assert.strictEqual(typeof hostPort, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appPortBindings WHERE hostPort=?', [ hostPort ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
 function del(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');

src/apps.js

@@ -118,7 +118,7 @@ AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name
 function validateHostname(location, fqdn) {
-    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, constants.MAIL_LOCATION, constants.POSTMAN_LOCATION ];
+    var RESERVED_LOCATIONS = [ config.adminLocation(), constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, config.mailLocation(), constants.POSTMAN_LOCATION ];
 
     if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new AppsError(AppsError.BAD_FIELD, location + ' is reserved');
 
@@ -639,14 +639,6 @@ function update(appId, data, auditSource, callback) {
 
         newConfig.manifest = manifest;
 
-        // TODO: disallow portBindings when an app updates and let ports simply be disabled. the new ports
-        // might conflict when the update is actually carried out as we do not 'reserve' them in the db
-        if ('portBindings' in data) {
-            newConfig.portBindings = data.portBindings;
-            error = validatePortBindings(data.portBindings, newConfig.manifest.tcpPorts);
-            if (error) return callback(error);
-        }
-
         if ('icon' in data) {
             if (data.icon) {
                 if (!validator.isBase64(data.icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
@@ -1012,14 +1004,9 @@ function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is {
         if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return new Error('Major version change'); // major changes are blocking
 
         var newTcpPorts = newManifest.tcpPorts || { };
-        var oldTcpPorts = app.manifest.tcpPorts || { };
         var portBindings = app.portBindings; // this is never null
 
-        for (var env in newTcpPorts) {
-            if (!(env in oldTcpPorts)) return new Error(env + ' is required from user');
-        }
-
-        for (env in portBindings) {
+        for (var env in portBindings) {
             if (!(env in newTcpPorts)) return new Error(env + ' was in use but new update removes it');
         }
 

src/appstore.js

@@ -169,12 +169,12 @@ function sendAliveStatus(data, callback) {
                 },
                 mailConfig: {
                     enabled: result[settings.MAIL_CONFIG_KEY].enabled
-            },
-            mailRelay: {
-                provider: result[settings.MAIL_RELAY_KEY].provider
-            },
-            mailCatchAll: {
-                count: result[settings.CATCH_ALL_ADDRESS_KEY].length
+                },
+                mailRelay: {
+                    provider: result[settings.MAIL_RELAY_KEY].provider
+                },
+                mailCatchAll: {
+                    count: result[settings.CATCH_ALL_ADDRESS_KEY].length
                 },
                 autoupdatePattern: result[settings.AUTOUPDATE_PATTERN_KEY],
                 timeZone: result[settings.TIME_ZONE_KEY],
@@ -183,6 +183,7 @@ function sendAliveStatus(data, callback) {
             var data = {
                 domain: config.fqdn(),
                 version: config.version(),
+                adminFqdn: config.adminFqdn(),
                 provider: config.provider(),
                 backendSettings: backendSettings,
                 machine: {

src/apptask.js

@@ -35,6 +35,7 @@ var addons = require('./addons.js'),
     certificates = require('./certificates.js'),
     config = require('./config.js'),
     database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
@@ -616,6 +617,27 @@ function update(app, callback) {
         // only delete unused addons after backup
         addons.teardownAddons.bind(null, app, unusedAddons),
 
+        // free unused ports
+        function (next) {
+            // make sure we always have objects
+            var currentPorts = app.portBindings || {};
+            var newPorts = app.newConfig.manifest.tcpPorts || {};
+
+            async.each(Object.keys(currentPorts), function (portName, callback) {
+                if (newPorts[portName]) return callback(); // port still in use
+
+                appdb.delPortBinding(currentPorts[portName], function (error) {
+                    if (error && error.reason === DatabaseError.NOT_FOUND) console.error('Portbinding does not exist in database.');
+                    else if (error) return next(error);
+
+                    // also delete from app object for further processing (the db is updated in the next step)
+                    delete app.portBindings[portName];
+
+                    callback();
+                });
+            }, next);
+        },
+
         // switch over to the new config. manifest, memoryLimit, portBindings, appstoreId are updated here
         updateApp.bind(null, app, app.newConfig),
 

src/certificates.js

@@ -177,7 +177,7 @@ function renewAll(auditSource, callback) {
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app
+        allApps.push({ location: config.adminLocation() }); // inject fake webadmin app
 
         var expiringApps = [ ];
         for (var i = 0; i < allApps.length; i++) {
@@ -239,7 +239,7 @@ function renewAll(auditSource, callback) {
                     }
 
                     // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
-                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                    var configureFunc = app.location === config.adminLocation() ?
                         nginx.configureAdmin.bind(null, certFilePath, keyFilePath, constants.NGINX_ADMIN_CONFIG_FILE_NAME, config.adminFqdn())
                         : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
 

src/cloudron.js

@@ -238,7 +238,7 @@ function configureWebadmin(callback) {
     function configureNginx(error) {
         debug('configureNginx: dns update:%j', error);
 
-        certificates.ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
+        certificates.ensureCertificate({ location: config.adminLocation() }, function (error, certFilePath, keyFilePath) {
             if (error) return done(error);
 
             gWebadminStatus.tls = true;
@@ -417,6 +417,9 @@ function getConfig(callback) {
                     apiServerOrigin: config.apiServerOrigin(),
                     webServerOrigin: config.webServerOrigin(),
                     fqdn: config.fqdn(),
+                    adminLocation: config.adminLocation(),
+                    adminFqdn: config.adminFqdn(),
+                    mailFqdn: config.mailFqdn(),
                     version: config.version(),
                     update: updateChecker.getUpdateInfo(),
                     progress: progress.getAll(),
@@ -538,7 +541,7 @@ function addDnsRecords(ip, callback) {
     var dkimKey = readDkimPublicKeySync();
     if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
 
-    var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
+    var webadminRecord = { subdomain: config.adminLocation(), type: 'A', values: [ ip ] };
     // t=s limits the domainkey to this domain and not it's subdomains
     var dkimRecord = { subdomain: constants.DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
 
@@ -670,19 +673,19 @@ function doUpgrade(boxUpdateInfo, callback) {
         if (error) return upgradeError(error);
 
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
-          .query({ token: config.token() })
-          .send({ version: boxUpdateInfo.version })
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
-            if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
+            .query({ token: config.token() })
+            .send({ version: boxUpdateInfo.version })
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
+                if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
 
-            progress.set(progress.UPDATE, 10, 'Updating base system');
+                progress.set(progress.UPDATE, 10, 'Updating base system');
 
-            // no need to unlock since this is the last thing we ever do on this box
-            callback();
-            retire('upgrade');
-        });
+                // no need to unlock since this is the last thing we ever do on this box
+                callback();
+                retire('upgrade');
+            });
     });
 }
 
@@ -706,6 +709,7 @@ function doUpdate(boxUpdateInfo, callback) {
             apiServerOrigin: config.apiServerOrigin(),
             webServerOrigin: config.webServerOrigin(),
             fqdn: config.fqdn(),
+            adminLocation: config.adminLocation(),
             tlsCert: config.tlsCert(),
             tlsKey: config.tlsKey(),
             isCustomDomain: config.isCustomDomain(),
@@ -844,20 +848,20 @@ function doMigrate(options, callback) {
         debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region);
 
         superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send(options)
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return unlock(error); // network error
-            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+            .query({ token: config.token() })
+            .send(options)
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return unlock(error); // network error
+                if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+                if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+                if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
 
-            progress.set(progress.MIGRATE, 10, 'Migrating');
+                progress.set(progress.MIGRATE, 10, 'Migrating');
 
-            retire('migrate', _.pick(options, 'domain', 'size', 'region'));
-        });
+                retire('migrate', _.pick(options, 'domain', 'size', 'region'));
+            });
     });
 
     callback(null);

src/config.js

@@ -28,7 +28,9 @@ exports = module.exports = {
     adminOrigin: adminOrigin,
     internalAdminOrigin: internalAdminOrigin,
     sysadminOrigin: sysadminOrigin, // caas routes
+    adminLocation: adminLocation,
     adminFqdn: adminFqdn,
+    mailLocation: mailLocation,
     mailFqdn: mailFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
@@ -45,7 +47,6 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    constants = require('./constants.js'),
     fs = require('fs'),
     path = require('path'),
     safe = require('safetydance'),
@@ -79,6 +80,7 @@ function initConfig() {
     // setup defaults
     data.fqdn = 'localhost';
     data.zoneName = '';
+    data.adminLocation = 'my';
 
     data.token = null;
     data.version = null;
@@ -176,16 +178,24 @@ function appFqdn(location) {
     return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
 }
 
-function adminFqdn() {
-    return appFqdn(constants.ADMIN_LOCATION);
+function mailLocation() {
+    return get('adminLocation'); // not a typo! should be same as admin location until we figure out certificates
 }
 
 function mailFqdn() {
-    return appFqdn(constants.MAIL_LOCATION);
+    return appFqdn(mailLocation());
+}
+
+function adminLocation() {
+    return get('adminLocation');
+}
+
+function adminFqdn() {
+    return appFqdn(adminLocation());
 }
 
 function adminOrigin() {
-    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
+    return 'https://' + appFqdn(adminLocation());
 }
 
 function internalAdminOrigin() {
@@ -237,4 +247,4 @@ function tlsKey() {
 function hasIPv6() {
     const IPV6_PROC_FILE = '/proc/net/if_inet6';
     return fs.existsSync(IPV6_PROC_FILE);
-}
+}

src/constants.js

@@ -1,12 +1,9 @@
 'use strict';
 
- // default admin installation location. keep in sync with ADMIN_LOCATION in setup/start.sh and BOX_ADMIN_LOCATION in appstore constants.js
 exports = module.exports = {
-    ADMIN_LOCATION: 'my',
     API_LOCATION: 'api', // this is unused but reserved for future use (#403)
     SMTP_LOCATION: 'smtp',
     IMAP_LOCATION: 'imap',
-    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
     POSTMAN_LOCATION: 'postman', // used in dovecot bounces
 
     // These are combined into one array because users and groups become mailboxes

src/dns/digitalocean.js

@@ -10,7 +10,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/digitalocean'),
     dns = require('dns'),
     safe = require('safetydance'),
@@ -37,22 +37,22 @@ function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
         var url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
 
         superagent.get(url)
-          .set('Authorization', 'Bearer ' + dnsConfig.token)
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-            if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND, formatError(result)));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND, formatError(result)));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-            matchingRecords = matchingRecords.concat(result.body.domain_records.filter(function (record) {
-                return (record.type === type && record.name === subdomain);
-            }));
+                matchingRecords = matchingRecords.concat(result.body.domain_records.filter(function (record) {
+                    return (record.type === type && record.name === subdomain);
+                }));
 
-            nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
+                nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
 
-            iteratorDone();
-        });
+                iteratorDone();
+            });
     }, function () { return !!nextPage; }, function (error) {
         if (error) return callback(error);
 
@@ -98,37 +98,37 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
 
             if (i >= result.length) {
                 superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
-                  .set('Authorization', 'Bearer ' + dnsConfig.token)
-                  .send(data)
-                  .timeout(30 * 1000)
-                  .end(function (error, result) {
-                    if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-                    if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-                    if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
-                    if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .end(function (error, result) {
+                        if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
+                        if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-                    recordIds.push(safe.query(result.body, 'domain_record.id'));
+                        recordIds.push(safe.query(result.body, 'domain_record.id'));
 
-                    return callback(null);
-                });
+                        return callback(null);
+                    });
             } else {
                 superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + result[i].id)
-                  .set('Authorization', 'Bearer ' + dnsConfig.token)
-                  .send(data)
-                  .timeout(30 * 1000)
-                  .end(function (error, result) {
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .end(function (error, result) {
                     // increment, as we have consumed the record
-                    ++i;
+                        ++i;
 
-                    if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-                    if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-                    if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
-                    if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+                        if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
+                        if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-                    recordIds.push(safe.query(result.body, 'domain_record.id'));
+                        recordIds.push(safe.query(result.body, 'domain_record.id'));
 
-                    return callback(null);
-                });
+                        return callback(null);
+                    });
             }
         }, function (error, id) {
             if (error) return callback(error);
@@ -183,18 +183,18 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
         // FIXME we only handle the first one currently
 
         superagent.del(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + tmp[0].id)
-          .set('Authorization', 'Bearer ' + dnsConfig.token)
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-            if (result.statusCode === 404) return callback(null);
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                if (result.statusCode === 404) return callback(null);
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-            debug('del: done');
+                debug('del: done');
 
-            return callback(null);
-        });
+                return callback(null);
+            });
     });
 }
 
@@ -221,7 +221,7 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
             return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Domain nameservers are not set to Digital Ocean'));
         }
 
-        const name = constants.ADMIN_LOCATION + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
+        const name = config.adminLocation() + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
 
         upsert(credentials, zoneName, name, 'A', [ ip ], function (error, changeId) {
             if (error) return callback(error);

src/dns/gcdns.js

@@ -9,10 +9,10 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    GCDNS = require('@google-cloud/dns'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/gcdns'),
     dns = require('dns'),
+    GCDNS = require('@google-cloud/dns'),
     SubdomainError = require('../subdomains.js').SubdomainError,
     util = require('util'),
     _ = require('underscore');
@@ -187,7 +187,7 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
                 return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS'));
             }
 
-            const name = constants.ADMIN_LOCATION + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
+            const name = config.adminLocation() + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
 
             upsert(credentials, zoneName, name, 'A', [ ip ], function (error, changeId) {
                 if (error) return callback(error);

src/dns/manual.js

@@ -10,7 +10,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/manual'),
     dig = require('../dig.js'),
     dns = require('dns'),
@@ -58,7 +58,7 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var adminDomain = constants.ADMIN_LOCATION + '.' + domain;
+    var adminDomain = config.adminLocation() + '.' + domain;
 
     dns.resolveNs(zoneName, function (error, nameservers) {
         if (error || !nameservers) return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Unable to get nameservers'));

src/dns/route53.js

@@ -13,7 +13,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     AWS = require('aws-sdk'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/route53'),
     dns = require('dns'),
     SubdomainError = require('../subdomains.js').SubdomainError,
@@ -247,7 +247,7 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
                 return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Domain nameservers are not set to Route53'));
             }
 
-            const name = constants.ADMIN_LOCATION + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
+            const name = config.adminLocation() + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
 
             upsert(credentials, zoneName, name, 'A', [ ip ], function (error, changeId) {
                 if (error) return callback(error);

src/infra_version.js

@@ -7,7 +7,7 @@
 exports = module.exports = {
     // a major version makes all apps restore from backup. #451 must be fixed before we do this.
     // a minor version makes all apps re-configure themselves
-    'version': '48.6.0',
+    'version': '48.7.0',
 
     'baseImages': [ 'cloudron/base:0.10.0' ],
 
@@ -18,7 +18,7 @@ exports = module.exports = {
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.17.0' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.13.0' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.11.0' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.37.4' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.38.1' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.12.0' }
     }
 };

src/ldap.js

@@ -61,12 +61,74 @@ function getUsersWithAccessToApp(req, callback) {
     });
 }
 
+// helper function to deal with pagination
+function finalSend(results, req, res, next) {
+    var min = 0;
+    var max = results.length;
+    var cookie = null;
+    var pageSize = 0;
+
+    // check if this is a paging request, if so get the cookie for session info
+    req.controls.forEach(function (control) {
+        if (control.type === ldap.PagedResultsControl.OID) {
+            pageSize = control.value.size;
+            cookie = control.value.cookie;
+        }
+    });
+
+    function sendPagedResults(start, end) {
+        start = (start < min) ? min : start;
+        end = (end > max || end < min) ? max : end;
+        var i;
+
+        for (i = start; i < end; i++) {
+            res.send(results[i]);
+        }
+
+        return i;
+    }
+
+    if (cookie && Buffer.isBuffer(cookie)) {
+        // we have pagination
+        var first = min;
+        if (cookie.length !== 0) {
+            first = parseInt(cookie.toString(), 10);
+        }
+        var last = sendPagedResults(first, first + pageSize);
+
+        var resultCookie;
+        if (last < max) {
+            resultCookie = new Buffer(last.toString());
+        } else {
+            resultCookie = new Buffer('');
+        }
+
+        res.controls.push(new ldap.PagedResultsControl({
+            value: {
+                size: pageSize, // correctness not required here
+                cookie: resultCookie
+            }
+        }));
+    } else {
+        // no pagination simply send all
+        results.forEach(function (result) {
+            res.send(result);
+        });
+    }
+
+    // all done
+    res.end();
+    next();
+}
+
 function userSearch(req, res, next) {
     debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     getUsersWithAccessToApp(req, function (error, result) {
         if (error) return next(error);
 
+        var results = [];
+
         // send user objects
         result.forEach(function (entry) {
             // skip entries with empty username. Some apps like owncloud can't deal with this
@@ -109,11 +171,11 @@ function userSearch(req, res, next) {
             if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
             if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                res.send(obj);
+                results.push(obj);
             }
         });
 
-        res.end();
+        finalSend(results, req, res, next);
     });
 }
 
@@ -123,6 +185,8 @@ function groupSearch(req, res, next) {
     getUsersWithAccessToApp(req, function (error, result) {
         if (error) return next(error);
 
+        var results = [];
+
         var groups = [{
             name: 'users',
             admin: false
@@ -149,11 +213,43 @@ function groupSearch(req, res, next) {
             if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
             if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                res.send(obj);
+                results.push(obj);
             }
         });
 
-        res.end();
+        finalSend(results, req, res, next);
+    });
+}
+
+function groupUsersCompare(req, res, next) {
+    debug('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
+
+    getUsersWithAccessToApp(req, function (error, result) {
+        if (error) return next(error);
+
+        // we only support memberuid here, if we add new group attributes later add them here
+        if (req.attribute === 'memberuid') {
+            var found = result.find(function (u) { return u.id === req.value; });
+            if (found) return res.end(true);
+        }
+
+        res.end(false);
+    });
+}
+
+function groupAdminsCompare(req, res, next) {
+    debug('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
+
+    getUsersWithAccessToApp(req, function (error, result) {
+        if (error) return next(error);
+
+        // we only support memberuid here, if we add new group attributes later add them here
+        if (req.attribute === 'memberuid') {
+            var found = result.find(function (u) { return u.id === req.value; });
+            if (found && found.admin) return res.end(true);
+        }
+
+        res.end(false);
     });
 }
 
@@ -161,6 +257,7 @@ function mailboxSearch(req, res, next) {
     debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
     var name = req.dn.rdns[0].attrs.cn.value.toLowerCase();
     // allow login via email
     var parts = name.split('@');
@@ -188,9 +285,11 @@ function mailboxSearch(req, res, next) {
         var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
         if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
-        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
-
-        res.end();
+        if (lowerCaseFilter.matches(obj.attributes)) {
+            finalSend([ obj ], req, res, next);
+        } else {
+            res.end();
+        }
     });
 }
 
@@ -198,6 +297,7 @@ function mailAliasSearch(req, res, next) {
     debug('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
     mailboxdb.getAlias(req.dn.rdns[0].attrs.cn.value.toLowerCase(), function (error, alias) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
@@ -218,9 +318,11 @@ function mailAliasSearch(req, res, next) {
         var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
         if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
-        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
-
-        res.end();
+        if (lowerCaseFilter.matches(obj.attributes)) {
+            finalSend([ obj ], req, res, next);
+        } else {
+            res.end();
+        }
     });
 }
 
@@ -228,6 +330,7 @@ function mailingListSearch(req, res, next) {
     debug('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
     mailboxdb.getGroup(req.dn.rdns[0].attrs.cn.value.toLowerCase(), function (error, group) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
@@ -248,9 +351,11 @@ function mailingListSearch(req, res, next) {
         var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
         if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
-        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
-
-        res.end();
+        if (lowerCaseFilter.matches(obj.attributes)) {
+            finalSend([ obj ], req, res, next);
+        } else {
+            res.end();
+        }
     });
 }
 
@@ -370,6 +475,9 @@ function start(callback) {
     gServer.bind('ou=recvmail,dc=cloudron', authenticateMailbox);
     gServer.bind('ou=sendmail,dc=cloudron', authenticateMailbox);
 
+    gServer.compare('cn=users,ou=groups,dc=cloudron', groupUsersCompare);
+    gServer.compare('cn=admins,ou=groups,dc=cloudron', groupAdminsCompare);
+
     // this is the bind for addons (after bind, they might search and authenticate)
     gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
         debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id

src/mailer.js

@@ -385,7 +385,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
             var templateDataHTML = JSON.parse(JSON.stringify(templateData));
             templateDataHTML.format = 'html';
 
-             var mailOptions = {
+            var mailOptions = {
                 from: mailConfig().from,
                 to: adminEmails.join(', '),
                 subject: util.format('%s has a new update available', config.fqdn()),
@@ -405,7 +405,7 @@ function appUpdateAvailable(app, updateInfo) {
     getAdminEmails(function (error, adminEmails) {
         if (error) return console.log('Error getting admins', error);
 
-         var mailOptions = {
+        var mailOptions = {
             from: mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('[%s] Update available for %s', config.fqdn(), app.fqdn),
@@ -561,8 +561,8 @@ function sendFeedback(user, type, subject, description) {
         type === exports.FEEDBACK_TYPE_UPGRADE_REQUEST ||
         type === exports.FEEDBACK_TYPE_APP_ERROR);
 
-        var mailOptions = {
-            from: mailConfig().from,
+    var mailOptions = {
+        from: mailConfig().from,
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})

src/routes/apps.js

@@ -313,11 +313,10 @@ function updateApp(req, res, next) {
     if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
     if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
 
-    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
-    debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
+    debug('Update app id:%s to manifest:%j', req.params.id, data.manifest);
 
     apps.update(req.params.id, req.body, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));

src/routes/test/apps-test.js

@@ -344,10 +344,10 @@ describe('App API', function () {
     it('app install fails - reserved admin location', function (done) {
         superagent.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ manifest: APP_MANIFEST, location: constants.ADMIN_LOCATION, accessRestriction: null })
+               .send({ manifest: APP_MANIFEST, location: 'my', accessRestriction: null })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
-            expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
+            expect(res.body.message).to.eql('my is reserved');
             done();
         });
     });

src/settings.js

@@ -33,9 +33,6 @@ exports = module.exports = {
     getTlsConfig: getTlsConfig,
     setTlsConfig: setTlsConfig,
 
-    getUpdateConfig: getUpdateConfig,
-    setUpdateConfig: setUpdateConfig,
-
     getAppstoreConfig: getAppstoreConfig,
     setAppstoreConfig: setAppstoreConfig,
 
@@ -429,30 +426,6 @@ function setBackupConfig(backupConfig, callback) {
     });
 }
 
-function getUpdateConfig(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settingsdb.get(exports.UPDATE_CONFIG_KEY, function (error, value) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.UPDATE_CONFIG_KEY]);
-        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
-
-        callback(null, JSON.parse(value)); // { prerelease }
-    });
-}
-
-function setUpdateConfig(updateConfig, callback) {
-    assert.strictEqual(typeof updateConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    settingsdb.set(exports.UPDATE_CONFIG_KEY, JSON.stringify(updateConfig), function (error) {
-        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
-
-        exports.events.emit(exports.UPDATE_CONFIG_KEY, updateConfig);
-
-        callback(null);
-    });
-}
-
 function getMailConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 

src/storage/filesystem.js

@@ -163,14 +163,15 @@ function testConfig(apiConfig, callback) {
     if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') return callback(new BackupsError(BackupsError.BAD_FIELD, 'noHardlinks must be boolean'));
 
     fs.stat(apiConfig.backupFolder, function (error, result) {
-        if (error) {
-            debug('testConfig: %s', apiConfig.backupFolder, error);
-            return callback(new BackupsError(BackupsError.BAD_FIELD, 'Directory does not exist or cannot be accessed'));
-        }
-
+        if (error) return callback(new BackupsError(BackupsError.BAD_FIELD, 'Directory does not exist or cannot be accessed: ' + error.message));
         if (!result.isDirectory()) return callback(new BackupsError(BackupsError.BAD_FIELD, 'Backup location is not a directory'));
 
-        callback(null);
+        mkdirp(path.join(apiConfig.backupFolder, 'snapshot'), function (error) {
+            if (error && error.code === 'EACCES') return callback(new BackupsError(BackupsError.BAD_FIELD, `Access denied. Run "chown yellowtent:yellowtent ${apiConfig.backupFolder}" on the server`));
+            if (error) return callback(new BackupsError(BackupsError.BAD_FIELD, error.message));
+
+            callback(null);
+        });
     });
 }
 

src/storage/s3.js

@@ -108,7 +108,7 @@ function getS3Config(apiConfig, callback) {
 
     if (apiConfig.endpoint) credentials.endpoint = apiConfig.endpoint;
 
-    if (apiConfig.acceptSelfSignedCerts === true) {
+    if (apiConfig.acceptSelfSignedCerts === true && credentials.endpoint && credentials.endpoint.startsWith('https://')) {
         credentials.httpOptions.agent = {
             agent: new https.Agent({ rejectUnauthorized: false })
         };
@@ -190,7 +190,7 @@ function download(apiConfig, backupFilePath, callback) {
     });
 }
 
-function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callback) {
+function listDir(apiConfig, backupFilePath, iteratorCallback, callback) {
     getS3Config(apiConfig, function (error, credentials) {
         if (error) return callback(error);
 
@@ -207,10 +207,9 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
                     return foreverCallback(error);
                 }
 
-                var arr = batchSize === 1 ? listData.Contents : chunk(listData.Contents, batchSize);
-                if (arr.length === 0) return foreverCallback(new Error('Done'));
+                if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));
 
-                iteratorCallback(s3, arr, function (error) {
+                iteratorCallback(s3, listData.Contents, function (error) {
                     if (error) return foreverCallback(error);
 
                     if (!listData.IsTruncated) return foreverCallback(new Error('Done'));
@@ -262,9 +261,9 @@ function downloadDir(apiConfig, backupFilePath, destDir) {
         });
     }
 
-    const concurrency = 10, batchSize = 1;
+    const concurrency = 10;
 
-    listDir(apiConfig, backupFilePath, batchSize, function (s3, objects, done) {
+    listDir(apiConfig, backupFilePath, function (s3, objects, done) {
         total += objects.length;
         async.eachLimit(objects, concurrency, downloadFile.bind(null, s3), done);
     }, function (error) {
@@ -370,10 +369,9 @@ function copy(apiConfig, oldFilePath, newFilePath) {
         });
     }
 
-    const batchSize = 1;
     var total = 0, concurrency = 4;
 
-    listDir(apiConfig, oldFilePath, batchSize, function (s3, objects, done) {
+    listDir(apiConfig, oldFilePath, function (s3, objects, done) {
         total += objects.length;
 
         if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
@@ -429,7 +427,6 @@ function removeDir(apiConfig, pathPrefix) {
                 Objects: contents.map(function (c) { return { Key: c.Key }; })
             }
         };
-        total += contents.length;
 
         events.emit('progress', `Removing ${contents.length} files from ${contents[0].Key} to ${contents[contents.length-1].Key}`);
 
@@ -443,9 +440,14 @@ function removeDir(apiConfig, pathPrefix) {
         });
     }
 
-    const batchSize = apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; // throttle requests per second
+    listDir(apiConfig, pathPrefix, function (s3, objects, done) {
+        total += objects.length;
 
-    listDir(apiConfig, pathPrefix, batchSize, deleteFiles, function (error) {
+        const batchSize = apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; // throttle objects in each request
+        var chunks = batchSize === 1 ? objects : chunk(objects, batchSize);
+
+        async.eachSeries(chunks, deleteFiles.bind(null, s3), done);
+    }, function (error) {
         events.emit('progress', `Removed ${total} files`);
 
         events.emit('done', error);

src/test/apps-test.js

@@ -135,7 +135,7 @@ describe('Apps', function () {
 
     describe('validateHostname', function () {
         it('does not allow admin subdomain', function () {
-            expect(apps._validateHostname(constants.ADMIN_LOCATION, 'cloudron.us')).to.be.an(Error);
+            expect(apps._validateHostname('my', 'cloudron.us')).to.be.an(Error);
         });
 
         it('cannot have >63 length subdomains', function () {

src/test/config-test.js

@@ -39,7 +39,7 @@ describe('config', function () {
     it('did set default values', function () {
         expect(config.isCustomDomain()).to.equal(true);
         expect(config.fqdn()).to.equal('localhost');
-        expect(config.adminOrigin()).to.equal('https://' + constants.ADMIN_LOCATION + '.localhost');
+        expect(config.adminOrigin()).to.equal('https://my.localhost');
         expect(config.appFqdn('app')).to.equal('app.localhost');
         expect(config.zoneName()).to.equal('');
     });
@@ -68,7 +68,7 @@ describe('config', function () {
 
         expect(config.isCustomDomain()).to.equal(true);
         expect(config.fqdn()).to.equal('example.com');
-        expect(config.adminOrigin()).to.equal('https://' + constants.ADMIN_LOCATION + '.example.com');
+        expect(config.adminOrigin()).to.equal('https://my.example.com');
         expect(config.appFqdn('app')).to.equal('app.example.com');
         expect(config.zoneName()).to.equal('example.com');
     });
@@ -79,7 +79,7 @@ describe('config', function () {
 
         expect(config.isCustomDomain()).to.equal(false);
         expect(config.fqdn()).to.equal('test.example.com');
-        expect(config.adminOrigin()).to.equal('https://' + constants.ADMIN_LOCATION + '-test.example.com');
+        expect(config.adminOrigin()).to.equal('https://my-test.example.com');
         expect(config.appFqdn('app')).to.equal('app-test.example.com');
         expect(config.zoneName()).to.equal('example.com');
     });

src/test/ldap-test.js

@@ -342,6 +342,35 @@ describe('Ldap', function () {
             });
         });
 
+        it ('succeeds with pagination', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: 'objectcategory=person',
+                paged: true
+            };
+
+            client.search('ou=users,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+                    entries.sort(function (a, b) { return a.username > b.username; });
+                    expect(entries[0].username).to.equal(USER_0.username.toLowerCase());
+                    expect(entries[0].mail).to.equal(USER_0.email.toLowerCase());
+                    expect(entries[1].username).to.equal(USER_1.username.toLowerCase());
+                    expect(entries[1].mail).to.equal(USER_1.email.toLowerCase());
+                    done();
+                });
+            });
+        });
+
         it ('succeeds with basic filter and email enabled', function (done) {
             // user settingsdb instead of settings, to not trigger further events
             settingsdb.set(settings.MAIL_CONFIG_KEY, JSON.stringify({ enabled: true }), function (error) {
@@ -617,13 +646,50 @@ describe('Ldap', function () {
                 });
             });
         });
+
+        it ('succeeds with pagination', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: 'objectclass=group',
+                paged: true
+            };
+
+            client.search('ou=groups,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+
+                    // ensure order for testability
+                    entries.sort(function (a, b) { return a.username < b.username; });
+
+                    expect(entries[0].cn).to.equal('users');
+                    expect(entries[0].memberuid.length).to.equal(3);
+                    expect(entries[0].memberuid[0]).to.equal(USER_0.id);
+                    expect(entries[0].memberuid[1]).to.equal(USER_1.id);
+                    expect(entries[0].memberuid[2]).to.equal(USER_2.id);
+                    expect(entries[1].cn).to.equal('admins');
+                    // if only one entry, the array becomes a string :-/
+                    expect(entries[1].memberuid).to.equal(USER_0.id);
+                    done();
+                });
+            });
+        });
     });
 
     function ldapSearch(dn, filter, callback) {
         var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
 
         var opts = {
-            filter: filter
+            filter: filter,
+            paged: true
         };
 
         client.search(dn, opts, function (error, result) {

src/test/settings-test.js

@@ -162,29 +162,6 @@ describe('Settings', function () {
             });
         });
 
-        it('can get default update config config', function (done) {
-            settings.getUpdateConfig(function (error, updateConfig) {
-                expect(error).to.be(null);
-                expect(updateConfig.prerelease).to.be(false);
-                done();
-            });
-        });
-
-        it('can set update config', function (done) {
-            settings.setUpdateConfig({ prerelease: true }, function (error) {
-                expect(error).to.be(null);
-                done();
-            });
-        });
-
-        it('can get update config', function (done) {
-            settings.getUpdateConfig(function (error, updateConfig) {
-                expect(error).to.be(null);
-                expect(updateConfig.prerelease).to.be(true);
-                done();
-            });
-        });
-
         it('can set mail config', function (done) {
             settings.setMailConfig({ enabled: true }, function (error) {
                 expect(error).to.be(null);

src/test/updatechecker-test.js

@@ -66,7 +66,8 @@ describe('updatechecker - box - manual (email)', function () {
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settings.setAutoupdatePattern.bind(null, constants.AUTOUPDATE_PATTERN_NEVER),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' })),
-            mailer._clearMailQueue
+            mailer._clearMailQueue,
+            mailer.start
         ], done);
     });
 
@@ -113,7 +114,7 @@ describe('updatechecker - box - manual (email)', function () {
         });
     });
 
-    it('does not offer prerelease', function (done) {
+    it('offers prerelease', function (done) {
         nock.cleanAll();
 
         var scope = nock('http://localhost:4444')
@@ -121,39 +122,18 @@ describe('updatechecker - box - manual (email)', function () {
             .query({ boxVersion: config.version(), accessToken: 'token' })
             .reply(200, { version: '2.0.0-pre.0', changelog: [''], sourceTarballUrl: '2.0.0-pre.0.tar.gz' } );
 
+        var scope2 = nock('http://localhost:4444')
+            .get('/api/v1/users/uid/cloudrons/cid/subscription')
+            .query({ accessToken: 'token' })
+            .reply(200, { subscription: { plan: { id: 'pro' } } } );
+
         updatechecker.checkBoxUpdates(function (error) {
             expect(!error).to.be.ok();
-            expect(updatechecker.getUpdateInfo().box).to.be(null);
+            expect(updatechecker.getUpdateInfo().box.version).to.be('2.0.0-pre.0');
             expect(scope.isDone()).to.be.ok();
+            expect(scope2.isDone()).to.be.ok();
 
-            checkMails(0, done);
-        });
-    });
-
-    it('offers prerelease', function (done) {
-        nock.cleanAll();
-
-        settings.setUpdateConfig({ prerelease: true }, function (error) {
-            if (error) return done(error);
-
-            var scope = nock('http://localhost:4444')
-                .get('/api/v1/users/uid/cloudrons/cid/boxupdate')
-                .query({ boxVersion: config.version(), accessToken: 'token' })
-                .reply(200, { version: '2.0.0-pre.0', changelog: [''], sourceTarballUrl: '2.0.0-pre.0.tar.gz' } );
-
-            var scope2 = nock('http://localhost:4444')
-                .get('/api/v1/users/uid/cloudrons/cid/subscription')
-                .query({ accessToken: 'token' })
-                .reply(200, { subscription: { plan: { id: 'pro' } } } );
-
-            updatechecker.checkBoxUpdates(function (error) {
-                expect(!error).to.be.ok();
-                expect(updatechecker.getUpdateInfo().box.version).to.be('2.0.0-pre.0');
-                expect(scope.isDone()).to.be.ok();
-                expect(scope2.isDone()).to.be.ok();
-
-                checkMails(1, done);
-            });
+            checkMails(1, done);
         });
     });
 
@@ -184,6 +164,7 @@ describe('updatechecker - box - automatic (no email)', function () {
             database.initialize,
             settings.initialize,
             mailer._clearMailQueue,
+            mailer.start,
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' }))
         ], done);

src/updatechecker.js

@@ -19,7 +19,6 @@ var apps = require('./apps.js'),
     mailer = require('./mailer.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
-    semver = require('semver'),
     settings = require('./settings.js');
 
 var gAppUpdateInfo = { }, // id -> update info { creationDate, manifest }
@@ -149,48 +148,37 @@ function checkBoxUpdates(callback) {
     appstore.getBoxUpdate(function (error, updateInfo) {
         if (error || !updateInfo) return callback(error);
 
-        settings.getUpdateConfig(function (error, updateConfig) {
+        gBoxUpdateInfo = updateInfo;
+
+        // decide whether to send email
+        var state = loadState();
+
+        if (state.box === gBoxUpdateInfo.version) {
+            debug('Skipping notification of box update as user was already notified');
+            return callback();
+        }
+
+        appstore.getSubscription(function (error, result) {
             if (error) return callback(error);
 
-            var isPrerelease = semver.parse(updateInfo.version).prerelease.length !== 0;
-
-            if (isPrerelease && !updateConfig.prerelease) {
-                debug('Skipping update %s since this box does not want prereleases', updateInfo.version);
-                return callback();
+            function done() {
+                state.box = updateInfo.version;
+                saveState(state);
+                callback();
             }
 
-            gBoxUpdateInfo = updateInfo;
-
-            // decide whether to send email
-            var state = loadState();
-
-            if (state.box === gBoxUpdateInfo.version) {
-                debug('Skipping notification of box update as user was already notified');
-                return callback();
+            // always send notifications if user is on the free plan
+            if (result.plan.id === 'free' || result.plan.id === 'undecided') {
+                mailer.boxUpdateAvailable(updateInfo.version, updateInfo.changelog);
+                return done();
             }
 
-            appstore.getSubscription(function (error, result) {
-                if (error) return callback(error);
+            // only send notifications if update pattern is 'never'
+            settings.getAutoupdatePattern(function (error, result) {
+                if (error) debug(error);
+                else if (result === constants.AUTOUPDATE_PATTERN_NEVER) mailer.boxUpdateAvailable(updateInfo.version, updateInfo.changelog);
 
-                function done() {
-                    state.box = updateInfo.version;
-                    saveState(state);
-                    callback();
-                }
-
-                // always send notifications if user is on the free plan
-                if (result.plan.id === 'free' || result.plan.id === 'undecided') {
-                    mailer.boxUpdateAvailable(updateInfo.version, updateInfo.changelog);
-                    return done();
-                }
-
-                // only send notifications if update pattern is 'never'
-                settings.getAutoupdatePattern(function (error, result) {
-                    if (error) debug(error);
-                    else if (result === constants.AUTOUPDATE_PATTERN_NEVER) mailer.boxUpdateAvailable(updateInfo.version, updateInfo.changelog);
-
-                    done();
-                });
+                done();
             });
         });
     });

webadmin/src/js/client.js

@@ -366,10 +366,9 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
         }).error(defaultErrorHandler(callback));
     };
 
-    Client.prototype.updateApp = function (id, manifest, portBindings, callback) {
+    Client.prototype.updateApp = function (id, manifest, callback) {
         var data =  {
-            appStoreId: manifest.id + '@' + manifest.version,
-            portBindings: portBindings
+            appStoreId: manifest.id + '@' + manifest.version
         };
 
         post('/api/v1/apps/' + id + '/update', data).success(function (data, status) {

webadmin/src/views/apps.html

@@ -179,13 +179,13 @@
             <div class="modal-header">
                 <h4 class="modal-title">Restore {{ appRestore.app.fqdn }}</h4>
             </div>
-            <div class="modal-body" ng-show="appRestore.backups.length === 0 && appRestore.busy">
+            <div class="modal-body" ng-show="appRestore.busyFetching">
                 <h4 class="text-center"><i class="fa fa-circle-o-notch fa-spin"></i> Fetching backups</h4>
             </div>
-            <div class="modal-body" ng-show="appRestore.backups.length === 0 && !appRestore.busy">
+            <div class="modal-body" ng-show="appRestore.backups.length === 0 && !appRestore.busyFetching">
                 <h4 class="text-danger">This app has no backups.</h4>
             </div>
-            <div class="modal-body" ng-show="appRestore.backups.length !== 0 && !appRestore.busy">
+            <div class="modal-body" ng-show="appRestore.backups.length !== 0">
                 <p>Restoring the app will lose all content generated since the backup.</p>
                 <label class="control-label">Select backup</label>
                 <div class="dropdown">
@@ -198,7 +198,7 @@
                 </div>
                 <br/>
                 <fieldset>
-                    <form role="form" name="appRestoreForm" ng-submit="doRestore()" autocomplete="off">
+                    <form role="form" name="appRestoreForm" ng-submit="appRestore.submit()" autocomplete="off">
                         <div class="form-group" ng-class="{ 'has-error': (appRestoreForm.password.$dirty && appRestoreForm.password.$invalid) || (!appRestoreForm.password.$dirty && appRestore.error.password) }">
                             <label class="control-label" for="appRestorePasswordInput">Provide your password to confirm this action</label>
                             <div class="control-label" ng-show="(appRestoreForm.password.$dirty && appRestoreForm.password.$invalid) || (!appRestoreForm.password.$dirty && appRestore.error.password)">
@@ -214,7 +214,7 @@
             </div>
             <div class="modal-footer">
                 <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
-                <button type="button" class="btn btn-success" ng-click="doRestore()" ng-show="appRestore.backups.length !== 0" ng-disabled="appRestoreForm.$invalid || appRestore.busy || !appRestore.selectedBackup"><i class="fa fa-circle-o-notch fa-spin" ng-show="appRestore.busy"></i> Restore</button>
+                <button type="button" class="btn btn-success" ng-click="appRestore.submit()" ng-show="appRestore.backups.length !== 0" ng-disabled="appRestoreForm.$invalid || appRestore.busy || !appRestore.selectedBackup"><i class="fa fa-circle-o-notch fa-spin" ng-show="appRestore.busy"></i> Restore</button>
             </div>
         </div>
     </div>
@@ -299,29 +299,6 @@
             <div class="modal-body">
                 <p>Recent Changes for new version <b>{{ appUpdate.manifest.version}}</b>:</p>
                 <div ng-bind-html="appUpdate.manifest.changelog | markdown2html"></div>
-                <br/>
-                <fieldset>
-                    <form role="form" name="appUpdateForm" ng-submit="doUpdate(appUpdateForm)" autocomplete="off">
-                        <div ng-repeat="(env, info) in appUpdate.portBindingsInfo" ng-class="{ 'newPort': info.isNew }">
-                            <ng-form name="portInfo_form">
-                                <div class="form-group" ng-class="{ 'has-error': portInfo_form.itemName{{$index}}.$dirty && portInfo_form.itemName{{$index}}.$invalid }">
-                                    <label class="control-label" for="inputPortInfo{{env}}"><input type="checkbox" ng-model="appUpdate.portBindingsEnabled[env]"> <span ng-show="info.isNew">New - </span> {{ info.description }} ({{ HOST_PORT_MIN }} - {{ HOST_PORT_MAX }})</label>
-                                    <input type="number" class="form-control" ng-model="appUpdate.portBindings[env]" ng-disabled="!appUpdate.portBindingsEnabled[env]" id="inputPortInfo{{env}}" later-name="itemName{{$index}}" min="{{HOST_PORT_MIN}}" max="{{HOST_PORT_MAX}}" required>
-                                </div>
-                            </ng-form>
-                        </div>
-                        <div ng-repeat="(env, port) in appUpdate.obsoletePortBindings" class="obsoletePort">
-                            <ng-form name="obsoletePortInfo_form">
-                                <div class="form-group">
-                                    Obsolete -
-                                    <label class="control-label">{{ env }}</label>
-                                    <input type="number" class="form-control" ng-model="port" disabled>
-                                </div>
-                            </ng-form>
-                        </div>
-                        <input class="ng-hide" type="submit" ng-disabled="appUpdateForm.$invalid || busy"/>
-                    </form>
-                </fieldset>
             </div>
             <div class="modal-footer">
                 <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
@@ -404,7 +381,7 @@
                     <div class="grid-item-bottom-mobile" ng-show="user.admin">
                         <div class="row">
                             <div class="col-xs-4 text-left">
-                                <a href="" ng-click="showRestore(app)" ng-show="backupConfig.provider !== 'noop'">
+                                <a href="" ng-click="appRestore.show(app)" ng-show="backupConfig.provider !== 'noop'">
                                     <i class="fa fa-undo scale"></i>
                                 </a>
 
@@ -427,7 +404,7 @@
                         </div>
 
                         <div>
-                            <a href="" ng-click="showRestore(app)" ng-show="backupConfig.provider !== 'noop'" title="Restore App"><i class="fa fa-undo scale"></i></a>
+                            <a href="" ng-click="appRestore.show(app)" ng-show="backupConfig.provider !== 'noop'" title="Restore App"><i class="fa fa-undo scale"></i></a>
                         </div>
 
                         <div ng-show="(app.installationState === 'installed' || app.installationState === 'pending_configure') && !(app | installError)">

webadmin/src/views/apps.js

@@ -63,6 +63,7 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
 
     $scope.appRestore = {
         busy: false,
+        busyFetching: false,
         error: {},
         app: {},
         password: '',
@@ -71,6 +72,47 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
 
         selectBackup: function (backup) {
             $scope.appRestore.selectedBackup = backup;
+        },
+
+        show: function (app) {
+            $scope.reset();
+
+            $scope.appRestore.app = app;
+            $scope.appRestore.busyFetching = true;
+
+            $('#appRestoreModal').modal('show');
+
+            Client.getAppBackups(app.id, function (error, backups) {
+                if (error) {
+                    Client.error(error);
+                } else {
+                    $scope.appRestore.backups = backups;
+                    if (backups.length) $scope.appRestore.selectedBackup = backups[0]; // pre-select first backup
+                    $scope.appRestore.busyFetching = false;
+                }
+            });
+
+            return false; // prevent propagation and default
+        },
+
+        submit: function () {
+            $scope.appRestore.busy = true;
+            $scope.appRestore.error.password = null;
+
+            Client.restoreApp($scope.appRestore.app.id, $scope.appRestore.selectedBackup.id, $scope.appRestore.password, function (error) {
+                if (error && error.statusCode === 403) {
+                    $scope.appRestore.password = '';
+                    $scope.appRestore.error.password = true;
+                    $scope.appRestoreForm.password.$setPristine();
+                    $('#appRestorePasswordInput').focus();
+                } else if (error) {
+                    Client.error(error);
+                } else {
+                    $('#appRestoreModal').modal('hide');
+                }
+
+                $scope.appRestore.busy = false;
+            });
         }
     };
 
@@ -136,10 +178,6 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         $scope.appUpdate.error = {};
         $scope.appUpdate.app = {};
         $scope.appUpdate.manifest = {};
-        $scope.appUpdate.portBindings = {};
-
-        $scope.appUpdateForm.$setPristine();
-        $scope.appUpdateForm.$setUntouched();
 
         // reset restore dialog
         $scope.appRestore.error = {};
@@ -324,48 +362,6 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         return false; // prevent propagation and default
     };
 
-    $scope.showRestore = function (app) {
-        $scope.reset();
-
-        $scope.appRestore.app = app;
-        $scope.appRestore.busy = true;
-
-        $('#appRestoreModal').modal('show');
-
-        Client.getAppBackups(app.id, function (error, backups) {
-            if (error) {
-                Client.error(error);
-            } else {
-                $scope.appRestore.backups = backups;
-                if (backups.length) $scope.appRestore.selectedBackup = backups[0]; // pre-select first backup
-                $scope.appRestore.busy = false;
-            }
-        });
-
-        return false; // prevent propagation and default
-    };
-
-    $scope.doRestore = function () {
-        $scope.appRestore.busy = true;
-        $scope.appRestore.error.password = null;
-
-        Client.restoreApp($scope.appRestore.app.id, $scope.appRestore.selectedBackup.id, $scope.appRestore.password, function (error) {
-            if (error && error.statusCode === 403) {
-                $scope.appRestore.password = '';
-                $scope.appRestore.error.password = true;
-                $scope.appRestoreForm.password.$setPristine();
-                $('#appRestorePasswordInput').focus();
-            } else if (error) {
-                Client.error(error);
-            } else {
-                $('#appRestoreModal').modal('hide');
-                $scope.reset();
-            }
-
-            $scope.appRestore.busy = false;
-        });
-    };
-
     $scope.showUninstall = function (app) {
         $scope.reset();
 
@@ -408,82 +404,13 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         $scope.appUpdate.app = app;
         $scope.appUpdate.manifest = angular.copy(updateManifest);
 
-        // ensure we always operate on objects here
-        app.portBindings = app.portBindings || {};
-        app.manifest.tcpPorts = app.manifest.tcpPorts || {};
-        updateManifest.tcpPorts = updateManifest.tcpPorts || {};
-
-        // Activate below two lines for testing the UI
-        // updateManifest.tcpPorts['TEST_HTTP'] = { defaultValue: 1337, description: 'HTTP server'};
-        // app.manifest.tcpPorts['TEST_FOOBAR'] = { defaultValue: 1338, description: 'FOOBAR server'};
-        // app.portBindings['TEST_SSH'] = 1339;
-
-        var portBindingsInfo = {};                  // Portbinding map only for information
-        var portBindings = {};                      // This is the actual model holding the env:port pair
-        var portBindingsEnabled = {};               // This is the actual model holding the enabled/disabled flag
-        var obsoletePortBindings = {};              // Info map for obsolete port bindings, this is for display use only and thus not in the model
-        var portsChanged = false;
-        var env;
-
-        // detect new portbindings and copy all from manifest.tcpPorts
-        for (env in updateManifest.tcpPorts) {
-            portBindingsInfo[env] = updateManifest.tcpPorts[env];
-            if (!app.manifest.tcpPorts[env]) {
-                portBindingsInfo[env].isNew = true;
-                portBindingsEnabled[env] = true;
-
-                // use default integer port value in model
-                portBindings[env] = updateManifest.tcpPorts[env].defaultValue || 0;
-
-                portsChanged = true;
-            } else {
-                // detect if the port binding was enabled
-                if (app.portBindings[env]) {
-                    portBindings[env] = app.portBindings[env];
-                    portBindingsEnabled[env] = true;
-                } else {
-                    portBindings[env] = updateManifest.tcpPorts[env].defaultValue || 0;
-                    portBindingsEnabled[env] = false;
-                }
-            }
-        }
-
-        // detect obsolete portbindings (mappings in app.portBindings, but not anymore in updateManifest.tcpPorts)
-        for (env in app.manifest.tcpPorts) {
-            // only list the port if it is not in the new manifest and was enabled previously
-            if (!updateManifest.tcpPorts[env] && app.portBindings[env]) {
-                obsoletePortBindings[env] = app.portBindings[env];
-                portsChanged = true;
-            }
-        }
-
-        // now inject the maps into the $scope, we only show those if ports have changed
-        $scope.appUpdate.portBindings = portBindings;                 // always inject the model, so it gets used in the actual update call
-        $scope.appUpdate.portBindingsEnabled = portBindingsEnabled;   // always inject the model, so it gets used in the actual update call
-
-        if (portsChanged) {
-            $scope.appUpdate.portBindingsInfo = portBindingsInfo;
-            $scope.appUpdate.obsoletePortBindings = obsoletePortBindings;
-        } else {
-            $scope.appUpdate.portBindingsInfo = {};
-            $scope.appUpdate.obsoletePortBindings = {};
-        }
-
         $('#appUpdateModal').modal('show');
     };
 
     $scope.doUpdate = function (form) {
         $scope.appUpdate.busy = true;
 
-        // only use enabled ports from portBindings
-        var finalPortBindings = {};
-        for (var env in $scope.appUpdate.portBindings) {
-            if ($scope.appUpdate.portBindingsEnabled[env]) {
-                finalPortBindings[env] = $scope.appUpdate.portBindings[env];
-            }
-        }
-
-        Client.updateApp($scope.appUpdate.app.id, $scope.appUpdate.manifest, finalPortBindings, function (error) {
+        Client.updateApp($scope.appUpdate.app.id, $scope.appUpdate.manifest, function (error) {
             if (error) {
                 Client.error(error);
             } else {

webadmin/src/views/certs.html

@@ -33,7 +33,7 @@
                         <div class="form-group" ng-class="{ 'has-error': false }" ng-show="dnsCredentials.provider === 'gcdns'">
                             <div class="input-group">
                                 <input type="file" id="gcdnsKeyFileInput" style="display:none"/>
-                                <input type="text" class="form-control" placeholder="Service Account Key" ng-model="dnsCredentials.gcdnsKey.keyFileName" id="gcdnsKeyInput" name="cert" onclick="getElementById('gcdnsKeyFileInput').click();" style="cursor: pointer;" ng-disabled="dnsCredentials.busy" required>
+                                <input type="text" class="form-control" placeholder="Service Account Key" ng-model="dnsCredentials.gcdnsKey.keyFileName" id="gcdnsKeyInput" name="cert" onclick="getElementById('gcdnsKeyFileInput').click();" style="cursor: pointer;" ng-disabled="dnsCredentials.busy" ng-required="dnsCredentials.provider === 'gcdns'">
                                 <span class="input-group-addon">
                                   <i class="fa fa-upload" onclick="getElementById('gcdnsKeyFileInput').click();"></i>
                               </span>
@@ -90,7 +90,7 @@
                 </p>
 
                 <p ng-show="dnsCredentials.provider === 'manual'">
-                    Setup an <i>A</i> record for <b>my.{{ dnsCredentials.customDomain || 'example.com' }}</b> to this server's IP. All DNS records have to be setup manually <i>before</i> each app installation.
+                    Setup an <i>A</i> record for <b>{{ config.adminLocation }}.{{ dnsCredentials.customDomain || 'example.com' }}</b> to this server's IP. All DNS records have to be setup manually <i>before</i> each app installation.
                 </p>
             </div>
             <div class="modal-footer ">

webadmin/src/views/email.html

@@ -65,9 +65,9 @@
        <a href="" data-toggle="collapse" data-parent="#accordion" data-target="#mail_settings">Mail server settings for email clients</a>
        <div id="mail_settings" class="panel-collapse collapse">
          <br/>
-         <p><b>Incoming Mail (IMAP)</b><br/>Server: <span ng-click-select>my.{{config.fqdn}}</span><br/>Port: 993 (TLS)</p>
-         <p><b>Outgoing Mail (SMTP)</b><br/>Server: <span ng-click-select>my.{{config.fqdn}}</span><br/>Port: 587 (STARTTLS)</p>
-         <p><b>ManageSieve</b><br/>Server: <span ng-click-select>my.{{config.fqdn}}</span><br/>Port: 4190 (TLS)</p>
+         <p><b>Incoming Mail (IMAP)</b><br/>Server: <span ng-click-select>{{config.mailFqdn}}</span><br/>Port: 993 (TLS)</p>
+         <p><b>Outgoing Mail (SMTP)</b><br/>Server: <span ng-click-select>{{config.mailFqdn}}</span><br/>Port: 587 (STARTTLS)</p>
+         <p><b>ManageSieve</b><br/>Server: <span ng-click-select>{{config.mailFqdn}}</span><br/>Port: 4190 (TLS)</p>
          <p>All the servers require your Cloudron credentials for authentication.</p>
         </div>
       </div>