Fix update mail templates

LE contacts the server by hostname and not by IP. This means that
when installing and reconfiguring the app it hits the default_server
route since nginx configs for the app are not generated at.

When doing in the daily cert renew, the nginx configs exist and we
are unable to renew the certs.

CHANGES

@@ -1064,3 +1064,41 @@
 * Fix crash in carbon which made graphs disappear on some Cloudrons
 * Fix issue where OAuth SSO did not work when alternate domain was used
 
+[1.7.4]
+* Add rsync format for backups. This feature allows incremental backups
+* Add Google DNS backend (thanks @syn)
+* Add DigitalOcean spaces backup storage backend
+* Add Cloudscale and Exoscale as supported VPS providers
+* Display backup progress and status in the web interface
+* Preliminary IPv6 support
+* Add IP RBL status to web interface
+* Add auto-update pattern `Every wednesday night`
+* Update Haraka to 2.8.15. This fixes the issue where emails were bounced with the message 'Send MAIL FROM first'
+* Do not overwrite existing subdomain when app's location is changed
+* Add button to send test email
+* Fix crash in carbon which made graphs disappear on some Cloudrons
+* Fix issue where OAuth SSO did not work when alternate domain was used
+* Changelog is now rendered in markdown format
+
+[1.7.5]
+* Expose a TLS relay port from mail container for Go applications
+
+[1.7.6]
+* Port bindings cannot be configured in update route anymore
+* Implement LDAP group compare
+* Pre-releases are now offered by appstore and not handled in box code anymore
+* LDAP pagination support. This will fix the warnings in NextCloud and Rocket.Chat
+* Check if directories can be created in the backup directory
+* Do not set the HTTPS agent when using HTTP with minio backup backend
+* Fix regression where a new domain config could not be set in the UI
+* New mail container release that fixes email sending with SOGo
+* Show 404 page for unknown domains
+
+[1.7.7]
+* Allow setting app memory till memory limit
+* Make the dkim selector dynamic
+* Fix issue where app update dialog did not close
+* Fix LE cert renewal failures
+* Send user and cert info in digest emails
+* Send oom, app failures and other important mails to cloudron owner's alt mail
+

scripts/cloudron-setup

@@ -45,6 +45,7 @@ fi
 initBaseImage="true"
 # provisioning data
 domain=""
+adminLocation="my"
 zoneName=""
 provider=""
 encryptionKey=""
@@ -63,13 +64,14 @@ baseDataDir=""
 # TODO this is still there for the restore case, see other occasions below
 versionsUrl="https://s3.amazonaws.com/prod-cloudron-releases/versions.json"
 
-args=$(getopt -o "" -l "domain:,help,skip-baseimage-init,data:,data-dir:,provider:,encryption-key:,restore-url:,tls-provider:,version:,dns-provider:,env:,prerelease,skip-reboot,source-url:" -n "$0" -- "$@")
+args=$(getopt -o "" -l "domain:,help,skip-baseimage-init,data:,data-dir:,provider:,encryption-key:,restore-url:,tls-provider:,version:,dns-provider:,env:,admin-location:,prerelease,skip-reboot,source-url:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
     case "$1" in
     --domain) domain="$2"; shift 2;;
-    --help) echo "See https://cloudron.io/references/selfhosting.html on how to install Cloudron"; exit 0;;
+    --admin-location) adminLocation="$2"; shift 2;;
+    --help) echo "See https://cloudron.io/documentation/installation/ on how to install Cloudron"; exit 0;;
     --provider) provider="$2"; shift 2;;
     --encryption-key) encryptionKey="$2"; shift 2;;
     --restore-url) restoreUrl="$2"; shift 2;;
@@ -105,12 +107,12 @@ done
 # validate arguments in the absence of data
 if [[ -z "${dataJson}" ]]; then
     if [[ -z "${provider}" ]]; then
-        echo "--provider is required (azure, cloudscale, digitalocean, ec2, exoscale, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic)"
+        echo "--provider is required (azure, cloudscale.ch, digitalocean, ec2, exoscale, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic)"
         exit 1
     elif [[  \
                 "${provider}" != "ami" && \
                 "${provider}" != "azure" && \
-                "${provider}" != "cloudscale" && \
+                "${provider}" != "cloudscale.ch" && \
                 "${provider}" != "digitalocean" && \
                 "${provider}" != "ec2" && \
                 "${provider}" != "exoscale" && \
@@ -123,7 +125,7 @@ if [[ -z "${dataJson}" ]]; then
                 "${provider}" != "vultr" && \
                 "${provider}" != "generic" \
             ]]; then
-        echo "--provider must be one of: azure, cloudscale, digitalocean, ec2, exoscale, gce, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic"
+        echo "--provider must be one of: azure, cloudscale.ch, digitalocean, ec2, exoscale, gce, lightsail, linode, ovh, rosehosting, scaleway, vultr or generic"
         exit 1
     fi
 
@@ -197,6 +199,7 @@ if [[ -z "${dataJson}" ]]; then
     {
         "boxVersionsUrl": "${versionsUrl}",
         "fqdn": "${domain}",
+        "adminLocation": "${adminLocation}",
         "zoneName": "${zoneName}",
         "provider": "${provider}",
         "apiServerOrigin": "${apiServerOrigin}",
@@ -214,9 +217,6 @@ if [[ -z "${dataJson}" ]]; then
             "format": "tgz",
             "retentionSecs": 172800
         },
-        "updateConfig": {
-            "prerelease": ${prerelease}
-        },
         "version": "${version}"
     }
 EOF
@@ -226,6 +226,7 @@ EOF
     {
         "boxVersionsUrl": "${versionsUrl}",
         "fqdn": "${domain}",
+        "adminLocation": "${adminLocation}",
         "zoneName": "${zoneName}",
         "provider": "${provider}",
         "apiServerOrigin": "${apiServerOrigin}",

setup/argparser.sh

@@ -6,6 +6,7 @@ json="${source_dir}/../node_modules/.bin/json"
 # IMPORTANT: Fix cloudron.js:doUpdate if you add/remove any arg. keep these sorted for readability
 arg_api_server_origin=""
 arg_fqdn=""
+arg_admin_location=""
 arg_zone_name=""
 arg_is_custom_domain="false"
 arg_restore_key=""
@@ -20,7 +21,6 @@ arg_version=""
 arg_web_server_origin=""
 arg_backup_config=""
 arg_dns_config=""
-arg_update_config=""
 arg_provider=""
 arg_app_bundle=""
 arg_is_demo="false"
@@ -46,13 +46,16 @@ while true; do
         arg_is_custom_domain=$(echo "$2" | $json isCustomDomain)
         [[ "${arg_is_custom_domain}" == "" ]] && arg_is_custom_domain="true"
 
+        arg_admin_location=$(echo "$2" | $json adminLocation)
+        [[ "${arg_admin_location}" == "" ]] && arg_admin_location="my"
+
         # only update/restore have this valid (but not migrate)
         arg_api_server_origin=$(echo "$2" | $json apiServerOrigin)
         [[ "${arg_api_server_origin}" == "" ]] && arg_api_server_origin="https://api.cloudron.io"
         arg_web_server_origin=$(echo "$2" | $json webServerOrigin)
         [[ "${arg_web_server_origin}" == "" ]] && arg_web_server_origin="https://cloudron.io"
 
-        # TODO check if an where this is used
+        # TODO check if and where this is used
         arg_version=$(echo "$2" | $json version)
 
         # read possibly empty parameters here
@@ -86,9 +89,6 @@ while true; do
         arg_dns_config=$(echo "$2" | $json dnsConfig)
         [[ "${arg_dns_config}" == "null" ]] && arg_dns_config=""
 
-        arg_update_config=$(echo "$2" | $json updateConfig)
-        [[ "${arg_update_config}" == "null" ]] && arg_update_config=""
-
         shift 2
         ;;
     --) break;;

setup/splashpage.sh

@@ -7,7 +7,6 @@ readonly SETUP_WEBSITE_DIR="/home/yellowtent/setup/website"
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 readonly box_src_dir="$(realpath ${script_dir}/..)"
 readonly PLATFORM_DATA_DIR="/home/yellowtent/platformdata"
-readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
 echo "Setting up nginx update page"
 
@@ -19,7 +18,7 @@ fi
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
 
 # keep this is sync with config.js appFqdn()
-admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${ADMIN_LOCATION}.${arg_fqdn}" ||  echo "${ADMIN_LOCATION}-${arg_fqdn}")
+admin_fqdn=$([[ "${arg_is_custom_domain}" == "true" ]] && echo "${arg_admin_location}.${arg_fqdn}" ||  echo "${arg_admin_location}-${arg_fqdn}")
 admin_origin="https://${admin_fqdn}"
 
 # copy the website

setup/start.sh

@@ -95,11 +95,6 @@ mkdir -p "${BOX_DATA_DIR}/mail/dkim"
 mkdir -p /var/backups
 chmod 777 /var/backups
 
-echo "==> Migrating mail data"
-if [[ -d "${PLATFORM_DATA_DIR}/mail" ]]; then
-    find "${PLATFORM_DATA_DIR}/mail" -mindepth 1 -maxdepth 1 -exec mv --target-directory="${BOX_DATA_DIR}/mail" '{}' +
-fi
-
 echo "==> Configuring journald"
 sed -e "s/^#SystemMaxUse=.*$/SystemMaxUse=100M/" \
     -e "s/^#ForwardToSyslog=.*$/ForwardToSyslog=no/" \
@@ -191,7 +186,11 @@ if [[ ! -f /etc/mysql/mysql.cnf ]] || ! diff -q "${script_dir}/start/mysql.cnf"
         echo "Waiting for mysql jobs..."
         sleep 1
     done
-    systemctl restart mysql
+    while true; do
+        if systemctl restart mysql; then break; fi
+        echo "Restarting MySql again after sometime since this fails randomly"
+        sleep 1
+    done
 else
     systemctl start mysql
 fi
@@ -243,6 +242,24 @@ cd "${BOX_SRC_DIR}"
 BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up
 EOF
 
+echo "==> Adding automated configs"
+mysql -u root -p${mysql_root_password} -e "REPLACE INTO settings (name, value) VALUES (\"domain\", '{ \"fqdn\": \"$arg_fqdn\", \"zoneName\": \"$arg_zone_name\", \"adminLocation\": \"$arg_admin_location\" }')" box
+
+if [[ ! -z "${arg_backup_config}" ]]; then
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
+fi
+
+if [[ ! -z "${arg_dns_config}" ]]; then
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
+fi
+
+if [[ ! -z "${arg_tls_config}" ]]; then
+    mysql -u root -p${mysql_root_password} \
+        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
+fi
+
 echo "==> Creating cloudron.conf"
 cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
 {
@@ -251,6 +268,7 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "apiServerOrigin": "${arg_api_server_origin}",
     "webServerOrigin": "${arg_web_server_origin}",
     "fqdn": "${arg_fqdn}",
+    "adminLocation": "${arg_admin_location}",
     "zoneName": "${arg_zone_name}",
     "isCustomDomain": ${arg_is_custom_domain},
     "provider": "${arg_provider}",
@@ -289,31 +307,13 @@ fi
 echo "==> Changing ownership"
 chown "${USER}:${USER}" -R "${CONFIG_DIR}"
 chown "${USER}:${USER}" -R "${PLATFORM_DATA_DIR}/nginx" "${PLATFORM_DATA_DIR}/collectd" "${PLATFORM_DATA_DIR}/logrotate.d" "${PLATFORM_DATA_DIR}/addons" "${PLATFORM_DATA_DIR}/acme" "${PLATFORM_DATA_DIR}/backup"
-chown "${USER}:${USER}" -R "${BOX_DATA_DIR}"
-chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}/INFRA_VERSION" 2>/dev/null || true
 chown "${USER}:${USER}" "${PLATFORM_DATA_DIR}"
 
-echo "==> Adding automated configs"
-if [[ ! -z "${arg_backup_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"backup_config\", '$arg_backup_config')" box
-fi
-
-if [[ ! -z "${arg_dns_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"dns_config\", '$arg_dns_config')" box
-fi
-
-if [[ ! -z "${arg_update_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"update_config\", '$arg_update_config')" box
-fi
-
-if [[ ! -z "${arg_tls_config}" ]]; then
-    mysql -u root -p${mysql_root_password} \
-        -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
-fi
+# do not chown the boxdata/mail directory; dovecot gets upset
+chown "${USER}:${USER}" "${BOX_DATA_DIR}"
+find "${BOX_DATA_DIR}" -mindepth 1 -maxdepth 1 -not -path "${BOX_DATA_DIR}/mail" -exec chown -R "${USER}:${USER}" {} \;
+chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
 
 set_progress "60" "Starting Cloudron"
 systemctl start cloudron.target

setup/start/nginx/appconfig.ejs

@@ -4,6 +4,41 @@ map $http_upgrade $connection_upgrade {
     '' close;
 }
 
+# http server
+server {
+    listen       80;
+<% if (hasIPv6) { -%>
+    listen       [::]:80;
+<% } -%>
+
+<% if (vhost) { -%>
+    server_name  <%= vhost %>;
+<% } else { -%>
+    # IP based access from collectd or initial cloudron setup. TODO: match the IPv6 address
+    server_name  "~^\d+\.\d+\.\d+\.\d+$";
+
+    # collectd
+    location /nginx_status {
+        stub_status on;
+        access_log off;
+        allow 127.0.0.1;
+        deny all;
+    }
+<% } -%>
+
+    # acme challenges (for cert renewal where the vhost config exists)
+    location /.well-known/acme-challenge/ {
+        default_type text/plain;
+        alias /home/yellowtent/platformdata/acme/;
+    }
+
+    location / {
+        # redirect everything to HTTPS
+        return 301 https://$host$request_uri;
+    }
+}
+
+# https server
 server {
 <% if (vhost) { -%>
     server_name  <%= vhost %>;

setup/start/nginx/nginx.conf

@@ -36,28 +36,21 @@ http {
     # zones for rate limiting
     limit_req_zone $binary_remote_addr zone=admin_login:10m rate=10r/s; # 10 request a second
 
-    # HTTP server
+
+    # default http server that returns 404 for any domain we are not listening on
     server {
-        listen      80;
-        listen      [::]:80;
+        listen      80 default_server;
+        listen      [::]:80 default_server;
+        server_name does_not_match_anything;
 
-        # collectd
-        location /nginx_status {
-            stub_status on;
-            access_log off;
-            allow 127.0.0.1;
-            deny all;
-        }
-
-        # acme challenges
+        # acme challenges (for app installation and re-configure when the vhost config does not exist)
         location /.well-known/acme-challenge/ {
             default_type text/plain;
             alias /home/yellowtent/platformdata/acme/;
         }
 
         location / {
-            # redirect everything to HTTPS
-            return 301 https://$host$request_uri;
+            return 404;
         }
     }
 

src/addons.js

@@ -363,6 +363,7 @@ function setupSendMail(app, options, callback) {
         var env = [
             { name: 'MAIL_SMTP_SERVER', value: 'mail' },
             { name: 'MAIL_SMTP_PORT', value: '2525' },
+            { name: 'MAIL_SMTPS_PORT', value: '4650' },
             { name: 'MAIL_SMTP_USERNAME', value: mailbox.name },
             { name: 'MAIL_SMTP_PASSWORD', value: password },
             { name: 'MAIL_FROM', value: mailbox.name + '@' + config.fqdn() },

src/appdb.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     update: update,
     getAll: getAll,
     getPortBindings: getPortBindings,
+    delPortBinding: delPortBinding,
 
     setAddonConfig: setAddonConfig,
     getAddonConfig: getAddonConfig,
@@ -252,6 +253,18 @@ function getPortBindings(id, callback) {
     });
 }
 
+function delPortBinding(hostPort, callback) {
+    assert.strictEqual(typeof hostPort, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM appPortBindings WHERE hostPort=?', [ hostPort ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
 function del(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');

src/apps.js

@@ -118,7 +118,7 @@ AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name
 function validateHostname(location, fqdn) {
-    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, constants.MAIL_LOCATION, constants.POSTMAN_LOCATION ];
+    var RESERVED_LOCATIONS = [ config.adminLocation(), constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, config.mailLocation(), constants.POSTMAN_LOCATION ];
 
     if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new AppsError(AppsError.BAD_FIELD, location + ' is reserved');
 
@@ -639,14 +639,6 @@ function update(appId, data, auditSource, callback) {
 
         newConfig.manifest = manifest;
 
-        // TODO: disallow portBindings when an app updates and let ports simply be disabled. the new ports
-        // might conflict when the update is actually carried out as we do not 'reserve' them in the db
-        if ('portBindings' in data) {
-            newConfig.portBindings = data.portBindings;
-            error = validatePortBindings(data.portBindings, newConfig.manifest.tcpPorts);
-            if (error) return callback(error);
-        }
-
         if ('icon' in data) {
             if (data.icon) {
                 if (!validator.isBase64(data.icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
@@ -1012,14 +1004,9 @@ function autoupdateApps(updateInfo, auditSource, callback) { // updateInfo is {
         if ((semver.major(app.manifest.version) !== 0) && (semver.major(app.manifest.version) !== semver.major(newManifest.version))) return new Error('Major version change'); // major changes are blocking
 
         var newTcpPorts = newManifest.tcpPorts || { };
-        var oldTcpPorts = app.manifest.tcpPorts || { };
         var portBindings = app.portBindings; // this is never null
 
-        for (var env in newTcpPorts) {
-            if (!(env in oldTcpPorts)) return new Error(env + ' is required from user');
-        }
-
-        for (env in portBindings) {
+        for (var env in portBindings) {
             if (!(env in newTcpPorts)) return new Error(env + ' was in use but new update removes it');
         }
 

src/appstore.js

@@ -164,16 +164,17 @@ function sendAliveStatus(data, callback) {
                     provider: result[settings.TLS_CONFIG_KEY].provider
                 },
                 backupConfig: {
-                    provider: result[settings.BACKUP_CONFIG_KEY].provider
+                    provider: result[settings.BACKUP_CONFIG_KEY].provider,
+                    hardlinks: !result[settings.BACKUP_CONFIG_KEY].noHardlinks
                 },
                 mailConfig: {
                     enabled: result[settings.MAIL_CONFIG_KEY].enabled
-            },
-            mailRelay: {
-                provider: result[settings.MAIL_RELAY_KEY].provider
-            },
-            mailCatchAll: {
-                count: result[settings.CATCH_ALL_ADDRESS_KEY].length
+                },
+                mailRelay: {
+                    provider: result[settings.MAIL_RELAY_KEY].provider
+                },
+                mailCatchAll: {
+                    count: result[settings.CATCH_ALL_ADDRESS_KEY].length
                 },
                 autoupdatePattern: result[settings.AUTOUPDATE_PATTERN_KEY],
                 timeZone: result[settings.TIME_ZONE_KEY],
@@ -182,6 +183,7 @@ function sendAliveStatus(data, callback) {
             var data = {
                 domain: config.fqdn(),
                 version: config.version(),
+                adminFqdn: config.adminFqdn(),
                 provider: config.provider(),
                 backendSettings: backendSettings,
                 machine: {

src/apptask.js

@@ -35,6 +35,7 @@ var addons = require('./addons.js'),
     certificates = require('./certificates.js'),
     config = require('./config.js'),
     database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
@@ -616,6 +617,27 @@ function update(app, callback) {
         // only delete unused addons after backup
         addons.teardownAddons.bind(null, app, unusedAddons),
 
+        // free unused ports
+        function (next) {
+            // make sure we always have objects
+            var currentPorts = app.portBindings || {};
+            var newPorts = app.newConfig.manifest.tcpPorts || {};
+
+            async.each(Object.keys(currentPorts), function (portName, callback) {
+                if (newPorts[portName]) return callback(); // port still in use
+
+                appdb.delPortBinding(currentPorts[portName], function (error) {
+                    if (error && error.reason === DatabaseError.NOT_FOUND) console.error('Portbinding does not exist in database.');
+                    else if (error) return next(error);
+
+                    // also delete from app object for further processing (the db is updated in the next step)
+                    delete app.portBindings[portName];
+
+                    callback();
+                });
+            }, next);
+        },
+
         // switch over to the new config. manifest, memoryLimit, portBindings, appstoreId are updated here
         updateApp.bind(null, app, app.newConfig),
 

src/backups.js

@@ -276,8 +276,8 @@ function saveFsMetadata(appDataDir, callback) {
     if (execFiles === null) return callback(safe.error);
 
     var metadata = {
-        emptyDirs: emptyDirs.trim().split('\n'),
-        execFiles: execFiles.trim().split('\n')
+        emptyDirs: emptyDirs.length === 0 ? [ ] : emptyDirs.trim().split('\n'),
+        execFiles: execFiles.length === 0 ? [ ] : execFiles.trim().split('\n')
     };
 
     if (!safe.fs.writeFileSync(`${appDataDir}/fsmetadata.json`, JSON.stringify(metadata, null, 4))) return callback(safe.error);
@@ -792,12 +792,12 @@ function backupBoxAndApps(auditSource, callback) {
 
             progress.set(progress.BACKUP, step * processed, 'Backing up system data');
 
-            backupBoxWithAppBackupIds(backupIds, timestamp, function (error, filename) {
+            backupBoxWithAppBackupIds(backupIds, timestamp, function (error, backupId) {
                 progress.set(progress.BACKUP, 100, error ? error.message : '');
 
-                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, filename: filename });
+                eventlog.add(eventlog.ACTION_BACKUP_FINISH, auditSource, { errorMessage: error ? error.message : null, backupId: backupId, timestamp: timestamp });
 
-                callback(error, filename);
+                callback(error, backupId);
             });
         });
     });

src/certificates.js

@@ -177,7 +177,7 @@ function renewAll(auditSource, callback) {
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        allApps.push({ location: constants.ADMIN_LOCATION }); // inject fake webadmin app
+        allApps.push({ location: config.adminLocation() }); // inject fake webadmin app
 
         var expiringApps = [ ];
         for (var i = 0; i < allApps.length; i++) {
@@ -239,7 +239,7 @@ function renewAll(auditSource, callback) {
                     }
 
                     // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback
-                    var configureFunc = app.location === constants.ADMIN_LOCATION ?
+                    var configureFunc = app.location === config.adminLocation() ?
                         nginx.configureAdmin.bind(null, certFilePath, keyFilePath, constants.NGINX_ADMIN_CONFIG_FILE_NAME, config.adminFqdn())
                         : nginx.configureApp.bind(null, app, certFilePath, keyFilePath);
 

src/cloudron.js

@@ -12,7 +12,7 @@ exports = module.exports = {
     dnsSetup: dnsSetup,
     getLogs: getLogs,
 
-    sendHeartbeat: sendHeartbeat,
+    sendCaasHeartbeat: sendCaasHeartbeat,
 
     updateToLatest: updateToLatest,
     reboot: reboot,
@@ -238,7 +238,7 @@ function configureWebadmin(callback) {
     function configureNginx(error) {
         debug('configureNginx: dns update:%j', error);
 
-        certificates.ensureCertificate({ location: constants.ADMIN_LOCATION }, function (error, certFilePath, keyFilePath) {
+        certificates.ensureCertificate({ location: config.adminLocation() }, function (error, certFilePath, keyFilePath) {
             if (error) return done(error);
 
             gWebadminStatus.tls = true;
@@ -417,6 +417,9 @@ function getConfig(callback) {
                     apiServerOrigin: config.apiServerOrigin(),
                     webServerOrigin: config.webServerOrigin(),
                     fqdn: config.fqdn(),
+                    adminLocation: config.adminLocation(),
+                    adminFqdn: config.adminFqdn(),
+                    mailFqdn: config.mailFqdn(),
                     version: config.version(),
                     update: updateChecker.getUpdateInfo(),
                     progress: progress.getAll(),
@@ -437,8 +440,8 @@ function getConfig(callback) {
     });
 }
 
-function sendHeartbeat() {
-    if (config.provider() !== 'caas') return;
+function sendCaasHeartbeat() {
+    assert(config.provider() === 'caas', 'Heartbeat is only sent for managed cloudrons');
 
     var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/heartbeat';
     superagent.post(url).query({ token: config.token(), version: config.version() }).timeout(30 * 1000).end(function (error, result) {
@@ -538,9 +541,9 @@ function addDnsRecords(ip, callback) {
     var dkimKey = readDkimPublicKeySync();
     if (!dkimKey) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, new Error('Failed to read dkim public key')));
 
-    var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
+    var webadminRecord = { subdomain: config.adminLocation(), type: 'A', values: [ ip ] };
     // t=s limits the domainkey to this domain and not it's subdomains
-    var dkimRecord = { subdomain: constants.DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
+    var dkimRecord = { subdomain: config.dkimSelector() + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
 
     var records = [ ];
     if (config.isCustomDomain()) {
@@ -670,19 +673,19 @@ function doUpgrade(boxUpdateInfo, callback) {
         if (error) return upgradeError(error);
 
         superagent.post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/upgrade')
-          .query({ token: config.token() })
-          .send({ version: boxUpdateInfo.version })
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
-            if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
+            .query({ token: config.token() })
+            .send({ version: boxUpdateInfo.version })
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return upgradeError(new Error('Network error making upgrade request: ' + error));
+                if (result.statusCode !== 202) return upgradeError(new Error(util.format('Server not ready to upgrade. statusCode: %s body: %j', result.status, result.body)));
 
-            progress.set(progress.UPDATE, 10, 'Updating base system');
+                progress.set(progress.UPDATE, 10, 'Updating base system');
 
-            // no need to unlock since this is the last thing we ever do on this box
-            callback();
-            retire('upgrade');
-        });
+                // no need to unlock since this is the last thing we ever do on this box
+                callback();
+                retire('upgrade');
+            });
     });
 }
 
@@ -706,6 +709,7 @@ function doUpdate(boxUpdateInfo, callback) {
             apiServerOrigin: config.apiServerOrigin(),
             webServerOrigin: config.webServerOrigin(),
             fqdn: config.fqdn(),
+            adminLocation: config.adminLocation(),
             tlsCert: config.tlsCert(),
             tlsKey: config.tlsKey(),
             isCustomDomain: config.isCustomDomain(),
@@ -844,20 +848,20 @@ function doMigrate(options, callback) {
         debug('migrate: domain: %s size %s region %s', options.domain, options.size, options.region);
 
         superagent
-          .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
-          .query({ token: config.token() })
-          .send(options)
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return unlock(error); // network error
-            if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
-            if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
-            if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
+            .post(config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/migrate')
+            .query({ token: config.token() })
+            .send(options)
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return unlock(error); // network error
+                if (result.statusCode === 409) return unlock(new CloudronError(CloudronError.BAD_STATE));
+                if (result.statusCode === 404) return unlock(new CloudronError(CloudronError.NOT_FOUND));
+                if (result.statusCode !== 202) return unlock(new CloudronError(CloudronError.EXTERNAL_ERROR, util.format('%s %j', result.status, result.body)));
 
-            progress.set(progress.MIGRATE, 10, 'Migrating');
+                progress.set(progress.MIGRATE, 10, 'Migrating');
 
-            retire('migrate', _.pick(options, 'domain', 'size', 'region'));
-        });
+                retire('migrate', _.pick(options, 'domain', 'size', 'region'));
+            });
     });
 
     callback(null);

src/config.js

@@ -17,6 +17,7 @@ exports = module.exports = {
     apiServerOrigin: apiServerOrigin,
     webServerOrigin: webServerOrigin,
     fqdn: fqdn,
+    zoneName: zoneName,
     setFqdn: setFqdn,
     token: token,
     version: version,
@@ -28,12 +29,14 @@ exports = module.exports = {
     adminOrigin: adminOrigin,
     internalAdminOrigin: internalAdminOrigin,
     sysadminOrigin: sysadminOrigin, // caas routes
+    adminLocation: adminLocation,
     adminFqdn: adminFqdn,
+    mailLocation: mailLocation,
     mailFqdn: mailFqdn,
     appFqdn: appFqdn,
-    zoneName: zoneName,
     setZoneName: setZoneName,
     hasIPv6: hasIPv6,
+    dkimSelector: dkimSelector,
 
     isDemo: isDemo,
 
@@ -45,7 +48,6 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    constants = require('./constants.js'),
     fs = require('fs'),
     path = require('path'),
     safe = require('safetydance'),
@@ -79,6 +81,7 @@ function initConfig() {
     // setup defaults
     data.fqdn = 'localhost';
     data.zoneName = '';
+    data.adminLocation = 'my';
 
     data.token = null;
     data.version = null;
@@ -176,16 +179,24 @@ function appFqdn(location) {
     return isCustomDomain() ? location + '.' + fqdn() : location + '-' + fqdn();
 }
 
-function adminFqdn() {
-    return appFqdn(constants.ADMIN_LOCATION);
+function mailLocation() {
+    return get('adminLocation'); // not a typo! should be same as admin location until we figure out certificates
 }
 
 function mailFqdn() {
-    return appFqdn(constants.MAIL_LOCATION);
+    return appFqdn(mailLocation());
+}
+
+function adminLocation() {
+    return get('adminLocation');
+}
+
+function adminFqdn() {
+    return appFqdn(adminLocation());
 }
 
 function adminOrigin() {
-    return 'https://' + appFqdn(constants.ADMIN_LOCATION);
+    return 'https://' + appFqdn(adminLocation());
 }
 
 function internalAdminOrigin() {
@@ -237,4 +248,10 @@ function tlsKey() {
 function hasIPv6() {
     const IPV6_PROC_FILE = '/proc/net/if_inet6';
     return fs.existsSync(IPV6_PROC_FILE);
-}
+}
+
+function dkimSelector() {
+    var loc = adminLocation();
+    return loc === 'my' ? 'cloudron' : `cloudron-${loc.replace(/\./g, '')}`;
+}
+

src/constants.js

@@ -1,12 +1,9 @@
 'use strict';
 
- // default admin installation location. keep in sync with ADMIN_LOCATION in setup/start.sh and BOX_ADMIN_LOCATION in appstore constants.js
 exports = module.exports = {
-    ADMIN_LOCATION: 'my',
     API_LOCATION: 'api', // this is unused but reserved for future use (#403)
     SMTP_LOCATION: 'smtp',
     IMAP_LOCATION: 'imap',
-    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
     POSTMAN_LOCATION: 'postman', // used in dovecot bounces
 
     // These are combined into one array because users and groups become mailboxes
@@ -36,8 +33,6 @@ exports = module.exports = {
 
     DEMO_USERNAME: 'cloudron',
 
-    DKIM_SELECTOR: 'cloudron',
-
     AUTOUPDATE_PATTERN_NEVER: 'never'
 };
 

src/cron.js

@@ -35,7 +35,7 @@ var gAliveJob = null, // send periodic stats
     gCleanupTokensJob = null,
     gDockerVolumeCleanerJob = null,
     gDynamicDNSJob = null,
-    gHeartbeatJob = null, // for CaaS health check
+    gCaasHeartbeatJob = null, // for CaaS health check
     gSchedulerSyncJob = null,
     gDigestEmailJob = null;
 
@@ -53,18 +53,20 @@ var AUDIT_SOURCE = { userId: null, username: 'cron' };
 function initialize(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    gHeartbeatJob = new CronJob({
-        cronTime: '00 */1 * * * *', // every minute
-        onTick: cloudron.sendHeartbeat,
-        start: false
-    });
-    // hack: send the first heartbeat only after we are running for 60 seconds
-    // required as we end up sending a heartbeat and then cloudron-setup reboots the server
-    setTimeout(function () {
-        if (!gHeartbeatJob) return; // already uninitalized
-        gHeartbeatJob.start();
-        cloudron.sendHeartbeat();
-    }, 1000 * 60);
+    if (config.provider() === 'caas') {
+        gCaasHeartbeatJob = new CronJob({
+            cronTime: '00 */1 * * * *', // every minute
+            onTick: cloudron.sendCaasHeartbeat,
+            start: false
+        });
+        // hack: send the first heartbeat only after we are running for 60 seconds
+        // required as we end up sending a heartbeat and then cloudron-setup reboots the server
+        setTimeout(function () {
+            if (!gCaasHeartbeatJob) return; // already uninitalized
+            gCaasHeartbeatJob.start();
+            cloudron.sendCaasHeartbeat();
+        }, 1000 * 60);
+    }
 
     var randomHourMinute = Math.floor(60*Math.random());
     gAliveJob = new CronJob({
@@ -252,8 +254,8 @@ function uninitialize(callback) {
     if (gAppUpdateCheckerJob) gAppUpdateCheckerJob.stop();
     gAppUpdateCheckerJob = null;
 
-    if (gHeartbeatJob) gHeartbeatJob.stop();
-    gHeartbeatJob = null;
+    if (gCaasHeartbeatJob) gCaasHeartbeatJob.stop();
+    gCaasHeartbeatJob = null;
 
     if (gAliveJob) gAliveJob.stop();
     gAliveJob = null;

src/developer.js

@@ -7,19 +7,15 @@ exports = module.exports = {
 
     isEnabled: isEnabled,
     setEnabled: setEnabled,
-    issueDeveloperToken: issueDeveloperToken,
-    getNonApprovedApps: getNonApprovedApps
+    issueDeveloperToken: issueDeveloperToken
 };
 
 var assert = require('assert'),
     clients = require('./clients.js'),
-    config = require('./config.js'),
     constants = require('./constants.js'),
-    debug = require('debug')('box:developer'),
     eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
     settings = require('./settings.js'),
-    superagent = require('superagent'),
     util = require('util');
 
 function DeveloperError(reason, errorOrMessage) {
@@ -84,19 +80,3 @@ function issueDeveloperToken(user, auditSource, callback) {
         callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
     });
 }
-
-function getNonApprovedApps(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    var url = config.apiServerOrigin() + '/api/v1/boxes/' + config.fqdn() + '/apps';
-    superagent.get(url).query({ token: config.token(), boxVersion: config.version() }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, error));
-        if (result.statusCode === 401 || result.statusCode === 403) {
-            debug('Failed to list apps in development. Appstore token invalid or missing. Returning empty list.', result.body);
-            return callback(null, []);
-        }
-        if (result.statusCode !== 200) return callback(new DeveloperError(DeveloperError.EXTERNAL_ERROR, util.format('App listing failed. %s %j', result.status, result.body)));
-
-        callback(null, result.body.apps || []);
-    });
-}

src/digest.js

@@ -33,31 +33,38 @@ function maybeSend(callback) {
 
             var hasSubscription = result && result.plan.id !== 'free' && result.plan.id !== 'undecided';
 
-            eventlog.getByActionLastWeek(eventlog.ACTION_APP_UPDATE, function (error, appUpdates) {
+            eventlog.getByCreationTime(new Date(new Date() - 7*86400000), function (error, events) {
                 if (error) return callback(error);
 
-                eventlog.getByActionLastWeek(eventlog.ACTION_UPDATE, function (error, boxUpdates) {
-                    if (error) return callback(error);
+                var appUpdates = events.filter(function (e) { return e.action === eventlog.ACTION_APP_UPDATE; }).map(function (e) { return e.data; });
+                var boxUpdates = events.filter(function (e) { return e.action === eventlog.ACTION_UPDATE; }).map(function (e) { return e.data; });
+                var certRenewals = events.filter(function (e) { return e.action === eventlog.ACTION_CERTIFICATE_RENEWAL; }).map(function (e) { return e.data; });
+                var usersAdded = events.filter(function (e) { return e.action === eventlog.ACTION_USER_ADD; }).map(function (e) { return e.data; });
+                var usersRemoved = events.filter(function (e) { return e.action === eventlog.ACTION_USER_REMOVE; }).map(function (e) { return e.data; });
+                var finishedBackups = events.filter(function (e) { return e.action === eventlog.ACTION_BACKUP_FINISH && !e.errorMessage; }).map(function (e) { return e.data; });
 
-                    var info = {
-                        hasSubscription: hasSubscription,
+                if (error) return callback(error);
 
-                        pendingAppUpdates: pendingAppUpdates,
-                        pendingBoxUpdate: updateInfo.box || null,
+                var info = {
+                    hasSubscription: hasSubscription,
 
-                        finishedAppUpdates: (appUpdates || []).map(function (e) { return e.data; }),
-                        finishedBoxUpdates: (boxUpdates || []).map(function (e) { return e.data; })
-                    };
+                    pendingAppUpdates: pendingAppUpdates,
+                    pendingBoxUpdate: updateInfo.box || null,
 
-                    if (info.pendingAppUpdates.length || info.pendingBoxUpdate || info.finishedAppUpdates.length || info.finishedBoxUpdates.length) {
-                        debug('maybeSend: sending digest email', info);
-                        mailer.sendDigest(info);
-                    } else {
-                        debug('maybeSend: nothing happened, NOT sending digest email');
-                    }
+                    finishedAppUpdates: appUpdates,
+                    finishedBoxUpdates: boxUpdates,
 
-                    callback();
-                });
+                    certRenewals: certRenewals,
+                    finishedBackups: finishedBackups, // only the successful backups
+                    usersAdded: usersAdded,
+                    usersRemoved: usersRemoved // unused because we don't have username to work with
+                };
+
+                // always send digest for backup failure notification
+                debug('maybeSend: sending digest email', info);
+                mailer.sendDigest(info);
+
+                callback();
             });
         });
     });

src/dns/digitalocean.js

@@ -10,7 +10,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/digitalocean'),
     dns = require('dns'),
     safe = require('safetydance'),
@@ -37,22 +37,22 @@ function getInternal(dnsConfig, zoneName, subdomain, type, callback) {
         var url = nextPage ? nextPage : DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records';
 
         superagent.get(url)
-          .set('Authorization', 'Bearer ' + dnsConfig.token)
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-            if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND, formatError(result)));
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                if (result.statusCode === 404) return callback(new SubdomainError(SubdomainError.NOT_FOUND, formatError(result)));
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-            matchingRecords = matchingRecords.concat(result.body.domain_records.filter(function (record) {
-                return (record.type === type && record.name === subdomain);
-            }));
+                matchingRecords = matchingRecords.concat(result.body.domain_records.filter(function (record) {
+                    return (record.type === type && record.name === subdomain);
+                }));
 
-            nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
+                nextPage = (result.body.links && result.body.links.pages) ? result.body.links.pages.next : null;
 
-            iteratorDone();
-        });
+                iteratorDone();
+            });
     }, function () { return !!nextPage; }, function (error) {
         if (error) return callback(error);
 
@@ -98,37 +98,37 @@ function upsert(dnsConfig, zoneName, subdomain, type, values, callback) {
 
             if (i >= result.length) {
                 superagent.post(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records')
-                  .set('Authorization', 'Bearer ' + dnsConfig.token)
-                  .send(data)
-                  .timeout(30 * 1000)
-                  .end(function (error, result) {
-                    if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-                    if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-                    if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
-                    if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .end(function (error, result) {
+                        if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
+                        if (result.statusCode !== 201) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-                    recordIds.push(safe.query(result.body, 'domain_record.id'));
+                        recordIds.push(safe.query(result.body, 'domain_record.id'));
 
-                    return callback(null);
-                });
+                        return callback(null);
+                    });
             } else {
                 superagent.put(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + result[i].id)
-                  .set('Authorization', 'Bearer ' + dnsConfig.token)
-                  .send(data)
-                  .timeout(30 * 1000)
-                  .end(function (error, result) {
+                    .set('Authorization', 'Bearer ' + dnsConfig.token)
+                    .send(data)
+                    .timeout(30 * 1000)
+                    .end(function (error, result) {
                     // increment, as we have consumed the record
-                    ++i;
+                        ++i;
 
-                    if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-                    if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-                    if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
-                    if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+                        if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                        if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                        if (result.statusCode === 422) return callback(new SubdomainError(SubdomainError.BAD_FIELD, result.body.message));
+                        if (result.statusCode !== 200) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-                    recordIds.push(safe.query(result.body, 'domain_record.id'));
+                        recordIds.push(safe.query(result.body, 'domain_record.id'));
 
-                    return callback(null);
-                });
+                        return callback(null);
+                    });
             }
         }, function (error, id) {
             if (error) return callback(error);
@@ -183,18 +183,18 @@ function del(dnsConfig, zoneName, subdomain, type, values, callback) {
         // FIXME we only handle the first one currently
 
         superagent.del(DIGITALOCEAN_ENDPOINT + '/v2/domains/' + zoneName + '/records/' + tmp[0].id)
-          .set('Authorization', 'Bearer ' + dnsConfig.token)
-          .timeout(30 * 1000)
-          .end(function (error, result) {
-            if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
-            if (result.statusCode === 404) return callback(null);
-            if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
-            if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
+            .set('Authorization', 'Bearer ' + dnsConfig.token)
+            .timeout(30 * 1000)
+            .end(function (error, result) {
+                if (error && !error.response) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, util.format('Network error %s', error.message)));
+                if (result.statusCode === 404) return callback(null);
+                if (result.statusCode === 403 || result.statusCode === 401) return callback(new SubdomainError(SubdomainError.ACCESS_DENIED, formatError(result)));
+                if (result.statusCode !== 204) return callback(new SubdomainError(SubdomainError.EXTERNAL_ERROR, formatError(result)));
 
-            debug('del: done');
+                debug('del: done');
 
-            return callback(null);
-        });
+                return callback(null);
+            });
     });
 }
 
@@ -221,7 +221,7 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
             return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Domain nameservers are not set to Digital Ocean'));
         }
 
-        const name = constants.ADMIN_LOCATION + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
+        const name = config.adminLocation() + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
 
         upsert(credentials, zoneName, name, 'A', [ ip ], function (error, changeId) {
             if (error) return callback(error);

src/dns/gcdns.js

@@ -9,10 +9,10 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    GCDNS = require('@google-cloud/dns'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/gcdns'),
     dns = require('dns'),
+    GCDNS = require('@google-cloud/dns'),
     SubdomainError = require('../subdomains.js').SubdomainError,
     util = require('util'),
     _ = require('underscore');
@@ -187,7 +187,7 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
                 return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Domain nameservers are not set to Google Cloud DNS'));
             }
 
-            const name = constants.ADMIN_LOCATION + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
+            const name = config.adminLocation() + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
 
             upsert(credentials, zoneName, name, 'A', [ ip ], function (error, changeId) {
                 if (error) return callback(error);

src/dns/manual.js

@@ -10,7 +10,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     async = require('async'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/manual'),
     dig = require('../dig.js'),
     dns = require('dns'),
@@ -58,7 +58,7 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    var adminDomain = constants.ADMIN_LOCATION + '.' + domain;
+    var adminDomain = config.adminLocation() + '.' + domain;
 
     dns.resolveNs(zoneName, function (error, nameservers) {
         if (error || !nameservers) return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Unable to get nameservers'));

src/dns/route53.js

@@ -13,7 +13,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     AWS = require('aws-sdk'),
-    constants = require('../constants.js'),
+    config = require('../config.js'),
     debug = require('debug')('box:dns/route53'),
     dns = require('dns'),
     SubdomainError = require('../subdomains.js').SubdomainError,
@@ -247,7 +247,7 @@ function verifyDnsConfig(dnsConfig, fqdn, zoneName, ip, callback) {
                 return callback(new SubdomainError(SubdomainError.BAD_FIELD, 'Domain nameservers are not set to Route53'));
             }
 
-            const name = constants.ADMIN_LOCATION + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
+            const name = config.adminLocation() + (fqdn === zoneName ? '' :  '.' + fqdn.slice(0, - zoneName.length - 1));
 
             upsert(credentials, zoneName, name, 'A', [ ip ], function (error, changeId) {
                 if (error) return callback(error);

src/docker.js

@@ -367,8 +367,6 @@ function getContainerIdByIp(ip, callback) {
     assert.strictEqual(typeof ip, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('get container by ip %s', ip);
-
     var docker = exports.connection;
 
     docker.listNetworks({}, function (error, result) {
@@ -390,8 +388,6 @@ function getContainerIdByIp(ip, callback) {
         }
         if (!containerId) return callback(new Error('No container with that ip'));
 
-        debug('found container %s with ip %s', containerId, ip);
-
         callback(null, containerId);
     });
 }

src/email.js

@@ -132,7 +132,7 @@ function verifyRelay(relay, callback) {
 
 function checkDkim(callback) {
     var dkim = {
-        domain: constants.DKIM_SELECTOR + '._domainkey.' + config.fqdn(),
+        domain: config.dkimSelector() + '._domainkey.' + config.fqdn(),
         type: 'TXT',
         expected: null,
         value: null,

src/eventlog.js

@@ -6,7 +6,7 @@ exports = module.exports = {
     add: add,
     get: get,
     getAllPaged: getAllPaged,
-    getByActionLastWeek: getByActionLastWeek,
+    getByCreationTime: getByCreationTime,
     cleanup: cleanup,
 
     // keep in sync with webadmin index.js filter and CLI tool
@@ -98,21 +98,21 @@ function getAllPaged(action, search, page, perPage, callback) {
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    eventlogdb.getAllPaged(action, search, page, perPage, function (error, boxes) {
+    eventlogdb.getAllPaged(action, search, page, perPage, function (error, events) {
         if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
 
-        callback(null, boxes);
+        callback(null, events);
     });
 }
 
-function getByActionLastWeek(action, callback) {
-    assert(typeof action === 'string' || action === null);
+function getByCreationTime(creationTime, callback) {
+    assert(util.isDate(creationTime));
     assert.strictEqual(typeof callback, 'function');
 
-    eventlogdb.getByActionLastWeek(action, function (error, boxes) {
+    eventlogdb.getByCreationTime(creationTime, function (error, events) {
         if (error) return callback(new EventLogError(EventLogError.INTERNAL_ERROR, error));
 
-        callback(null, boxes);
+        callback(null, events);
     });
 }
 
@@ -120,7 +120,7 @@ function cleanup(callback) {
     callback = callback || NOOP_CALLBACK;
 
     var d = new Date();
-    d.setDate(d.getDate() - 7); // 7 days ago
+    d.setDate(d.getDate() - 10); // 10 days ago
 
     // only cleanup high frequency events
     var actions = [

src/eventlogdb.js

@@ -3,7 +3,7 @@
 exports = module.exports = {
     get: get,
     getAllPaged: getAllPaged,
-    getByActionLastWeek: getByActionLastWeek,
+    getByCreationTime: getByCreationTime,
     add: add,
     count: count,
     delByCreationTime: delByCreationTime,
@@ -73,12 +73,12 @@ function getAllPaged(action, search, page, perPage, callback) {
     });
 }
 
-function getByActionLastWeek(action, callback) {
-    assert(typeof action === 'string' || action === null);
+function getByCreationTime(creationTime, callback) {
+    assert(util.isDate(creationTime));
     assert.strictEqual(typeof callback, 'function');
 
-    var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE action=? AND creationTime >= DATE_SUB(NOW(), INTERVAL 1 WEEK) ORDER BY creationTime DESC';
-    database.query(query, [ action ], function (error, results) {
+    var query = 'SELECT ' + EVENTLOGS_FIELDS + ' FROM eventlog WHERE creationTime >= ?';
+    database.query(query, [ creationTime ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         results.forEach(postProcess);

src/infra_version.js

@@ -7,7 +7,7 @@
 exports = module.exports = {
     // a major version makes all apps restore from backup. #451 must be fixed before we do this.
     // a minor version makes all apps re-configure themselves
-    'version': '48.6.0',
+    'version': '48.8.0',
 
     'baseImages': [ 'cloudron/base:0.10.0' ],
 
@@ -18,7 +18,7 @@ exports = module.exports = {
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.17.0' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.13.0' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.11.0' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.37.3' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.39.0' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.12.0' }
     }
 };

src/ldap.js

@@ -61,12 +61,74 @@ function getUsersWithAccessToApp(req, callback) {
     });
 }
 
+// helper function to deal with pagination
+function finalSend(results, req, res, next) {
+    var min = 0;
+    var max = results.length;
+    var cookie = null;
+    var pageSize = 0;
+
+    // check if this is a paging request, if so get the cookie for session info
+    req.controls.forEach(function (control) {
+        if (control.type === ldap.PagedResultsControl.OID) {
+            pageSize = control.value.size;
+            cookie = control.value.cookie;
+        }
+    });
+
+    function sendPagedResults(start, end) {
+        start = (start < min) ? min : start;
+        end = (end > max || end < min) ? max : end;
+        var i;
+
+        for (i = start; i < end; i++) {
+            res.send(results[i]);
+        }
+
+        return i;
+    }
+
+    if (cookie && Buffer.isBuffer(cookie)) {
+        // we have pagination
+        var first = min;
+        if (cookie.length !== 0) {
+            first = parseInt(cookie.toString(), 10);
+        }
+        var last = sendPagedResults(first, first + pageSize);
+
+        var resultCookie;
+        if (last < max) {
+            resultCookie = new Buffer(last.toString());
+        } else {
+            resultCookie = new Buffer('');
+        }
+
+        res.controls.push(new ldap.PagedResultsControl({
+            value: {
+                size: pageSize, // correctness not required here
+                cookie: resultCookie
+            }
+        }));
+    } else {
+        // no pagination simply send all
+        results.forEach(function (result) {
+            res.send(result);
+        });
+    }
+
+    // all done
+    res.end();
+    next();
+}
+
 function userSearch(req, res, next) {
     debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     getUsersWithAccessToApp(req, function (error, result) {
         if (error) return next(error);
 
+        var results = [];
+
         // send user objects
         result.forEach(function (entry) {
             // skip entries with empty username. Some apps like owncloud can't deal with this
@@ -109,11 +171,11 @@ function userSearch(req, res, next) {
             if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
             if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                res.send(obj);
+                results.push(obj);
             }
         });
 
-        res.end();
+        finalSend(results, req, res, next);
     });
 }
 
@@ -123,6 +185,8 @@ function groupSearch(req, res, next) {
     getUsersWithAccessToApp(req, function (error, result) {
         if (error) return next(error);
 
+        var results = [];
+
         var groups = [{
             name: 'users',
             admin: false
@@ -149,11 +213,43 @@ function groupSearch(req, res, next) {
             if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
             if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                res.send(obj);
+                results.push(obj);
             }
         });
 
-        res.end();
+        finalSend(results, req, res, next);
+    });
+}
+
+function groupUsersCompare(req, res, next) {
+    debug('group users compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
+
+    getUsersWithAccessToApp(req, function (error, result) {
+        if (error) return next(error);
+
+        // we only support memberuid here, if we add new group attributes later add them here
+        if (req.attribute === 'memberuid') {
+            var found = result.find(function (u) { return u.id === req.value; });
+            if (found) return res.end(true);
+        }
+
+        res.end(false);
+    });
+}
+
+function groupAdminsCompare(req, res, next) {
+    debug('group admins compare: dn %s, attribute %s, value %s (from %s)', req.dn.toString(), req.attribute, req.value, req.connection.ldap.id);
+
+    getUsersWithAccessToApp(req, function (error, result) {
+        if (error) return next(error);
+
+        // we only support memberuid here, if we add new group attributes later add them here
+        if (req.attribute === 'memberuid') {
+            var found = result.find(function (u) { return u.id === req.value; });
+            if (found && found.admin) return res.end(true);
+        }
+
+        res.end(false);
     });
 }
 
@@ -161,6 +257,7 @@ function mailboxSearch(req, res, next) {
     debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
     var name = req.dn.rdns[0].attrs.cn.value.toLowerCase();
     // allow login via email
     var parts = name.split('@');
@@ -188,9 +285,11 @@ function mailboxSearch(req, res, next) {
         var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
         if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
-        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
-
-        res.end();
+        if (lowerCaseFilter.matches(obj.attributes)) {
+            finalSend([ obj ], req, res, next);
+        } else {
+            res.end();
+        }
     });
 }
 
@@ -198,6 +297,7 @@ function mailAliasSearch(req, res, next) {
     debug('mail alias get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
     mailboxdb.getAlias(req.dn.rdns[0].attrs.cn.value.toLowerCase(), function (error, alias) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
@@ -218,9 +318,11 @@ function mailAliasSearch(req, res, next) {
         var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
         if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
-        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
-
-        res.end();
+        if (lowerCaseFilter.matches(obj.attributes)) {
+            finalSend([ obj ], req, res, next);
+        } else {
+            res.end();
+        }
     });
 }
 
@@ -228,6 +330,7 @@ function mailingListSearch(req, res, next) {
     debug('mailing list get: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
 
     if (!req.dn.rdns[0].attrs.cn) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
     mailboxdb.getGroup(req.dn.rdns[0].attrs.cn.value.toLowerCase(), function (error, group) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
@@ -248,9 +351,11 @@ function mailingListSearch(req, res, next) {
         var lowerCaseFilter = safe(function () { return ldap.parseFilter(req.filter.toString().toLowerCase()); }, null);
         if (!lowerCaseFilter) return next(new ldap.OperationsError(safe.error.toString()));
 
-        if (lowerCaseFilter.matches(obj.attributes)) res.send(obj);
-
-        res.end();
+        if (lowerCaseFilter.matches(obj.attributes)) {
+            finalSend([ obj ], req, res, next);
+        } else {
+            res.end();
+        }
     });
 }
 
@@ -370,6 +475,9 @@ function start(callback) {
     gServer.bind('ou=recvmail,dc=cloudron', authenticateMailbox);
     gServer.bind('ou=sendmail,dc=cloudron', authenticateMailbox);
 
+    gServer.compare('cn=users,ou=groups,dc=cloudron', groupUsersCompare);
+    gServer.compare('cn=admins,ou=groups,dc=cloudron', groupAdminsCompare);
+
     // this is the bind for addons (after bind, they might search and authenticate)
     gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
         debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id

src/mail_templates/app_update_available.ejs

@@ -2,13 +2,14 @@
 
 Dear Cloudron Admin,
 
-a new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
-
-The app will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
+A new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
 
 Changes:
 <%= updateInfo.manifest.changelog %>
 
+<% if (!hasSubscription) { -%>
+*Keep your Cloudron automatically up-to-date and secure by upgrading to a paid plan at* <%= webadminUrl %>/#/settings
+<% } -%>
 
 Powered by https://cloudron.io
 
@@ -16,4 +17,35 @@ Sent at: <%= new Date().toUTCString() %>
 
 <% } else { %>
 
+    <center>
+
+    <img src="<%= cloudronAvatarUrl %>" width="128px" height="128px"/>
+
+    <h3>Dear <%= cloudronName %> Admin,</h3>
+
+    <div style="width: 650px; text-align: left;">
+        <p>
+            A new version <%= updateInfo.manifest.version %> of the app '<%= app.manifest.title %>' installed at <%= app.fqdn %> is available!
+        </p>
+
+        <h5>Changelog:</h5>
+        <%- changelogHTML %>
+
+        <br/>
+
+        <% if (!hasSubscription) { %>
+        <p>Keep your Cloudron automatically up-to-date and secure by upgrading to a <a href="<%= webadminUrl %>/#/settings">paid plan</a>.</p>
+        <% } %>
+
+        <br/>
+    </div>
+
+    <div style="font-size: 10px; color: #333333; background: #ffffff;">
+        Powered by <a href="https://cloudron.io">Cloudron</a>.
+    </div>
+
+    </center>
+
+    <img src="https://analytics.cloudron.io/piwik.php?idsite=2&rec=1&e_c=CloudronEmail&e_a=update" style="border:0" alt="" />
+
 <% } %>

src/mail_templates/box_update_available.ejs

@@ -4,15 +4,18 @@ Dear <%= cloudronName %> Admin,
 
 Version <%= newBoxVersion %> for Cloudron <%= fqdn %> is now available!
 
-Your Cloudron will update automatically tonight. Alternately, update immediately at <%= webadminUrl %>.
-
 Changelog:
 <% for (var i = 0; i < changelog.length; i++) { %>
   * <%- changelog[i] %>
 <% } %>
 
-Thank you,
-your Cloudron
+<% if (!hasSubscription) { -%>
+*Keep your Cloudron automatically up-to-date and secure by upgrading to a paid plan at* <%= webadminUrl %>/#/settings
+<% } -%>
+
+Powered by https://cloudron.io
+
+Sent at: <%= new Date().toUTCString() %>
 
 <% } else { %>
 
@@ -27,11 +30,6 @@ your Cloudron
         Version <b><%= newBoxVersion %></b> for Cloudron <%= fqdn %> is now available!
     </p>
 
-    <p>
-        Your Cloudron will update automatically tonight.<br/>
-        Alternately, update immediately <a href="<%= webadminUrl %>">here</a>.
-    </p>
-
     <h5>Changelog:</h5>
     <ul>
     <% for (var i = 0; i < changelogHTML.length; i++) { %>
@@ -40,6 +38,11 @@ your Cloudron
     </ul>
 
     <br/>
+
+    <% if (!hasSubscription) { %>
+    <p>Keep your Cloudron automatically up-to-date and secure by upgrading to a <a href="<%= webadminUrl %>/#/settings">paid plan</a>.</p>
+    <% } %>
+
     <br/>
 </div>
 

src/mail_templates/digest.ejs

@@ -2,7 +2,19 @@
 
 Dear <%= cloudronName %> Admin,
 
-This is the weekly summary of activities on your Cloudron <%= fqdn %>.
+This is a summary of the activities on your Cloudron <%= fqdn %>.
+<% if (info.usersAdded.length) { -%>
+
+The following users were added:
+<% for (var i = 0; i < info.usersAdded.length; i++) { -%>
+    * <%- info.usersAdded[i].email %>
+<% }} -%>
+<% if (info.certRenewals.length) { -%>
+
+The certificates of the following apps was renewed:
+<% for (var i = 0; i < info.certRenewals.length; i++) { -%>
+    * <%- info.certRenewals[i].domain %> - <%- info.certRenewals[i].errorMessage || 'Success' %>
+<% }} -%>
 <% if (info.pendingBoxUpdate) { -%>
 
 Cloudron v<%- info.pendingBoxUpdate.version %> is available:
@@ -33,6 +45,14 @@ The following apps were updated:
 <% for (var j = 0; j < info.finishedAppUpdates[i].toManifest.changelog.trim().split('\n').length; j++) { -%>
         <%= info.finishedAppUpdates[i].toManifest.changelog.trim().split('\n')[j] %>
 <% }}} -%>
+<% if (info.finishedBackups.length) { -%>
+
+Last successful backup: <%- info.finishedBackups[0].backupId || info.finishedBackups[0].filename %>
+<% } else { -%>
+
+This Cloudron did **not** backup successfully in the last week!
+<% } -%>
+
 <% if (!info.hasSubscription) { -%>
 
 *Keep your Cloudron automatically up-to-date and secure by upgrading to a paid plan at* <%= webadminUrl %>/#/settings
@@ -52,9 +72,25 @@ Sent at: <%= new Date().toUTCString() %>
 
         <br/>
 
-        <p>Weekly summary of activities on your Cloudron <a href="<%= webadminUrl %>"><%= cloudronName %></a>:</p>
+        <p>This is a summary of the activities on your Cloudron <a href="<%= webadminUrl %>"><%= cloudronName %></a> last week.</p>
 
-        <br/>
+    <% if (info.usersAdded.length) { -%>
+        <p><b>The following users were added:</b></p>
+        <ul>
+        <% for (var i = 0; i < info.usersAdded.length; i++) { %>
+            <li><%- info.usersAdded[i].email %></li>
+        <% } %>
+        </ul>
+    <% } %>
+
+    <% if (info.certRenewals.length) { -%>
+        <p><b>The certificates of the following apps were renewed:</b></p>
+        <ul>
+            <% for (var i = 0; i < info.certRenewals.length; i++) { %>
+                <li><%- info.certRenewals[i].domain %> - <%- info.certRenewals[i].errorMessage || 'Success' %></li>
+            <% } %>
+        </ul>
+    <% } %>
 
     <% if (info.pendingBoxUpdate) { -%>
         <p><b>Cloudron v<%- info.pendingBoxUpdate.version %> is available:</b></p>
@@ -113,6 +149,12 @@ Sent at: <%= new Date().toUTCString() %>
         </ul>
     <% } %>
 
+    <% if (info.finishedBackups.length) { %>
+        <p><b>Last successful backup : </b> <%= info.finishedBackups[0].backupId || info.finishedBackups[0].filename %> </p>
+    <% } else { %>
+        <p><b>This Cloudron did not backup successfully in the last week!</b></p>
+    <% } %>
+
         <br/>
 
     <% if (!info.hasSubscription) { %>
@@ -123,12 +165,12 @@ Sent at: <%= new Date().toUTCString() %>
         <br/>
         <br/>
 
-        <p style="text-align: right;">
+        <center>
             <small>
                 Powered by <a href="https://cloudron.io">Cloudron</a><br/>
                 Sent on <%= new Date().toUTCString() %>
             </small>
-        </p>
+        </center>
     </div>
 </center>
 

src/mailer.js

@@ -171,6 +171,7 @@ function getAdminEmails(callback) {
         if (admins.length === 0) return callback(new Error('No admins on this cloudron')); // box not activated yet
 
         var adminEmails = [ ];
+        if (admins[0].alternateEmail) adminEmails.push(admins[0].alternateEmail);
         admins.forEach(function (admin) { adminEmails.push(admin.email); });
 
         callback(null, adminEmails);
@@ -244,7 +245,7 @@ function userAdded(user, inviteSent) {
     debug('Sending mail for userAdded %s including invite link', inviteSent ? 'not' : '');
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
@@ -341,7 +342,7 @@ function appDied(app) {
     debug('Sending mail for app %s @ %s died', app.id, app.fqdn);
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         var mailOptions = {
             from: mailConfig().from,
@@ -354,12 +355,13 @@ function appDied(app) {
     });
 }
 
-function boxUpdateAvailable(newBoxVersion, changelog) {
+function boxUpdateAvailable(hasSubscription, newBoxVersion, changelog) {
+    assert.strictEqual(typeof hasSubscription, 'boolean');
     assert.strictEqual(typeof newBoxVersion, 'string');
     assert(util.isArray(changelog));
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         settings.getCloudronName(function (error, cloudronName) {
             if (error) {
@@ -373,6 +375,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
                 fqdn: config.fqdn(),
                 webadminUrl: config.adminOrigin(),
                 newBoxVersion: newBoxVersion,
+                hasSubscription: hasSubscription,
                 changelog: changelog,
                 changelogHTML: changelog.map(function (e) { return converter.makeHtml(e); }),
                 cloudronName: cloudronName,
@@ -385,7 +388,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
             var templateDataHTML = JSON.parse(JSON.stringify(templateData));
             templateDataHTML.format = 'html';
 
-             var mailOptions = {
+            var mailOptions = {
                 from: mailConfig().from,
                 to: adminEmails.join(', '),
                 subject: util.format('%s has a new update available', config.fqdn()),
@@ -398,29 +401,13 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
     });
 }
 
-function appUpdateAvailable(app, updateInfo) {
+function appUpdateAvailable(app, hasSubscription, info) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof updateInfo, 'object');
-
-    getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
-
-         var mailOptions = {
-            from: mailConfig().from,
-            to: adminEmails.join(', '),
-            subject: util.format('[%s] Update available for %s', config.fqdn(), app.fqdn),
-            text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
-        };
-
-        enqueue(mailOptions);
-    });
-}
-
-function sendDigest(info) {
+    assert.strictEqual(typeof hasSubscription, 'boolean');
     assert.strictEqual(typeof info, 'object');
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         settings.getCloudronName(function (error, cloudronName) {
             if (error) {
@@ -428,34 +415,73 @@ function sendDigest(info) {
                 cloudronName = 'Cloudron';
             }
 
-            appstore.getAccount(function (error, appstoreProfile) {
-                if (error && error.reason !== AppstoreError.BILLING_REQUIRED) console.error(error);
-                if (appstoreProfile) adminEmails.push(appstoreProfile.email);
+            var converter = new showdown.Converter();
 
-                var templateData = {
-                    fqdn: config.fqdn(),
-                    webadminUrl: config.adminOrigin(),
-                    cloudronName: cloudronName,
-                    cloudronAvatarUrl: config.adminOrigin() + '/api/v1/cloudron/avatar',
-                    info: info
-                };
+            var templateData = {
+                fqdn: config.fqdn(),
+                webadminUrl: config.adminOrigin(),
+                hasSubscription: hasSubscription,
+                app: app,
+                updateInfo: info,
+                changelogHTML: converter.makeHtml(info.manifest.changelog),
+                cloudronName: cloudronName,
+                cloudronAvatarUrl: config.adminOrigin() + '/api/v1/cloudron/avatar'
+            };
 
-                var templateDataText = JSON.parse(JSON.stringify(templateData));
-                templateDataText.format = 'text';
+            var templateDataText = JSON.parse(JSON.stringify(templateData));
+            templateDataText.format = 'text';
 
-                var templateDataHTML = JSON.parse(JSON.stringify(templateData));
-                templateDataHTML.format = 'html';
+            var templateDataHTML = JSON.parse(JSON.stringify(templateData));
+            templateDataHTML.format = 'html';
 
-                var mailOptions = {
-                    from: mailConfig().from,
-                    to: adminEmails.join(', '),
-                    subject: util.format('[%s] Cloudron - Weekly activity digest', config.fqdn()),
-                    text: render('digest.ejs', templateDataText),
-                    html: render('digest.ejs', templateDataHTML)
-                };
+            var mailOptions = {
+                from: mailConfig().from,
+                to: adminEmails.join(', '),
+                subject: util.format('App %s has a new update available', app.fqdn),
+                text: render('app_update_available.ejs', templateDataText),
+                html: render('app_update_available.ejs', templateDataHTML)
+            };
 
-                enqueue(mailOptions);
-            });
+            enqueue(mailOptions);
+        });
+    });
+}
+
+function sendDigest(info) {
+    assert.strictEqual(typeof info, 'object');
+
+    getAdminEmails(function (error, adminEmails) {
+        if (error) return debug('Error getting admins', error);
+
+        settings.getCloudronName(function (error, cloudronName) {
+            if (error) {
+                debug(error);
+                cloudronName = 'Cloudron';
+            }
+
+            var templateData = {
+                fqdn: config.fqdn(),
+                webadminUrl: config.adminOrigin(),
+                cloudronName: cloudronName,
+                cloudronAvatarUrl: config.adminOrigin() + '/api/v1/cloudron/avatar',
+                info: info
+            };
+
+            var templateDataText = JSON.parse(JSON.stringify(templateData));
+            templateDataText.format = 'text';
+
+            var templateDataHTML = JSON.parse(JSON.stringify(templateData));
+            templateDataHTML.format = 'html';
+
+            var mailOptions = {
+                from: mailConfig().from,
+                to: adminEmails.join(', '),
+                subject: util.format('[%s] Cloudron - Weekly activity digest', config.fqdn()),
+                text: render('digest.ejs', templateDataText),
+                html: render('digest.ejs', templateDataHTML)
+            };
+
+            enqueue(mailOptions);
         });
     });
 }
@@ -464,7 +490,7 @@ function outOfDiskSpace(message) {
     assert.strictEqual(typeof message, 'string');
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         var mailOptions = {
             from: mailConfig().from,
@@ -481,7 +507,7 @@ function backupFailed(error) {
     var message = splatchError(error);
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         var mailOptions = {
             from: mailConfig().from,
@@ -499,7 +525,7 @@ function certificateRenewalError(domain, message) {
     assert.strictEqual(typeof message, 'string');
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         var mailOptions = {
             from: mailConfig().from,
@@ -517,7 +543,7 @@ function oomEvent(program, context) {
     assert.strictEqual(typeof context, 'string');
 
     getAdminEmails(function (error, adminEmails) {
-        if (error) return console.log('Error getting admins', error);
+        if (error) return debug('Error getting admins', error);
 
         var mailOptions = {
             from: mailConfig().from,
@@ -561,8 +587,8 @@ function sendFeedback(user, type, subject, description) {
         type === exports.FEEDBACK_TYPE_UPGRADE_REQUEST ||
         type === exports.FEEDBACK_TYPE_APP_ERROR);
 
-        var mailOptions = {
-            from: mailConfig().from,
+    var mailOptions = {
+        from: mailConfig().from,
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})

src/platform.js

@@ -254,7 +254,7 @@ function createMailConfig(callback) {
             var mailFromValidation = result[settings.MAIL_FROM_VALIDATION_KEY];
 
             if (!safe.fs.writeFileSync(paths.ADDON_CONFIG_DIR + '/mail/mail.ini',
-                `mail_domain=${fqdn}\nmail_server_name=${mailFqdn}\nalerts_from=${alertsFrom}\nalerts_to=${alertsTo}\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\n`, 'utf8')) {
+                `mail_domain=${fqdn}\nmail_server_name=${mailFqdn}\nalerts_from=${alertsFrom}\nalerts_to=${alertsTo}\ncatch_all=${catchAll}\nmail_from_validation=${mailFromValidation}\ndkim_selector=${config.dkimSelector()}\n`, 'utf8')) {
                 return callback(new Error('Could not create mail var file:' + safe.error.message));
             }
 

src/routes/apps.js

@@ -313,11 +313,10 @@ function updateApp(req, res, next) {
     if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
     if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
 
-    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
-    debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
+    debug('Update app id:%s to manifest:%j', req.params.id, data.manifest);
 
     apps.update(req.params.id, req.body, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));

src/routes/developer.js

@@ -4,8 +4,7 @@ exports = module.exports = {
     enabled: enabled,
     setEnabled: setEnabled,
     status: status,
-    login: login,
-    apps: apps
+    login: login
 };
 
 var developer = require('../developer.js'),
@@ -52,9 +51,3 @@ function login(req, res, next) {
   })(req, res, next);
 }
 
-function apps(req, res, next) {
-    developer.getNonApprovedApps(function (error, result) {
-        if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(200, { apps: result }));
-    });
-}

src/routes/test/apps-test.js

@@ -344,10 +344,10 @@ describe('App API', function () {
     it('app install fails - reserved admin location', function (done) {
         superagent.post(SERVER_URL + '/api/v1/apps/install')
                .query({ access_token: token })
-               .send({ manifest: APP_MANIFEST, location: constants.ADMIN_LOCATION, accessRestriction: null })
+               .send({ manifest: APP_MANIFEST, location: 'my', accessRestriction: null })
                .end(function (err, res) {
             expect(res.statusCode).to.equal(400);
-            expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
+            expect(res.body.message).to.eql('my is reserved');
             done();
         });
     });

src/server.js

@@ -59,26 +59,26 @@ function initializeExpressSync() {
     router.del = router.delete; // amend router.del for readability further on
 
     app
-       .use(middleware.timeout(REQUEST_TIMEOUT))
-       .use(json)
-       .use(urlencoded)
-       .use(middleware.cookieParser())
-       .use(middleware.cors({ origins: [ '*' ], allowCredentials: false }))
-       .use(middleware.session({
-           secret: hat(128), // we only use the session during oauth, and already have an in-memory session store, so we can safely change that during restarts
-           resave: true,
-           saveUninitialized: true,
-           cookie: {
-               path: '/',
-               httpOnly: true,
-               secure: process.env.BOX_ENV !== 'test',
-               maxAge: 600000
-           }
-       }))
-       .use(passport.initialize())
-       .use(passport.session())
-       .use(router)
-       .use(middleware.lastMile());
+        .use(middleware.timeout(REQUEST_TIMEOUT))
+        .use(json)
+        .use(urlencoded)
+        .use(middleware.cookieParser())
+        .use(middleware.cors({ origins: [ '*' ], allowCredentials: false }))
+        .use(middleware.session({
+            secret: hat(128), // we only use the session during oauth, and already have an in-memory session store, so we can safely change that during restarts
+            resave: true,
+            saveUninitialized: true,
+            cookie: {
+                path: '/',
+                httpOnly: true,
+                secure: process.env.BOX_ENV !== 'test',
+                maxAge: 600000
+            }
+        }))
+        .use(passport.initialize())
+        .use(passport.session())
+        .use(router)
+        .use(middleware.lastMile());
 
     // NOTE: these limits have to be in sync with nginx limits
     var FILE_SIZE_LIMIT = '256mb', // max file size that can be uploaded (see also client_max_body_size in nginx)
@@ -108,7 +108,6 @@ function initializeExpressSync() {
     router.post('/api/v1/developer', developerScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.developer.setEnabled);
     router.get ('/api/v1/developer', developerScope, routes.developer.enabled, routes.developer.status);
     router.post('/api/v1/developer/login', routes.developer.enabled, routes.developer.login);
-    router.get ('/api/v1/developer/apps', developerScope, routes.developer.enabled, routes.developer.apps);
 
     // cloudron routes
     router.get ('/api/v1/cloudron/config', cloudronScope, routes.cloudron.getConfig);
@@ -284,11 +283,11 @@ function initializeSysadminExpressSync() {
     router.del = router.delete; // amend router.del for readability further on
 
     app
-       .use(middleware.timeout(REQUEST_TIMEOUT))
-       .use(json)
-       .use(urlencoded)
-       .use(router)
-       .use(middleware.lastMile());
+        .use(middleware.timeout(REQUEST_TIMEOUT))
+        .use(json)
+        .use(urlencoded)
+        .use(router)
+        .use(middleware.lastMile());
 
     // Sysadmin routes
     router.post('/api/v1/backup', routes.sysadmin.backup);

src/settings.js

@@ -33,9 +33,6 @@ exports = module.exports = {
     getTlsConfig: getTlsConfig,
     setTlsConfig: setTlsConfig,
 
-    getUpdateConfig: getUpdateConfig,
-    setUpdateConfig: setUpdateConfig,
-
     getAppstoreConfig: getAppstoreConfig,
     setAppstoreConfig: setAppstoreConfig,
 
@@ -429,30 +426,6 @@ function setBackupConfig(backupConfig, callback) {
     });
 }
 
-function getUpdateConfig(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    settingsdb.get(exports.UPDATE_CONFIG_KEY, function (error, value) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, gDefaults[exports.UPDATE_CONFIG_KEY]);
-        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
-
-        callback(null, JSON.parse(value)); // { prerelease }
-    });
-}
-
-function setUpdateConfig(updateConfig, callback) {
-    assert.strictEqual(typeof updateConfig, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    settingsdb.set(exports.UPDATE_CONFIG_KEY, JSON.stringify(updateConfig), function (error) {
-        if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
-
-        exports.events.emit(exports.UPDATE_CONFIG_KEY, updateConfig);
-
-        callback(null);
-    });
-}
-
 function getMailConfig(callback) {
     assert.strictEqual(typeof callback, 'function');
 

src/storage/filesystem.js

@@ -107,7 +107,8 @@ function copy(apiConfig, oldFilePath, newFilePath) {
         if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
 
         // this will hardlink backups saving space
-        shell.exec('copy', '/bin/cp', [ '-al', oldFilePath, newFilePath ], { }, function (error) {
+        var cpOptions = apiConfig.noHardlinks ? '-a' : '-al';
+        shell.exec('copy', '/bin/cp', [ cpOptions, oldFilePath, newFilePath ], { }, function (error) {
             if (error) return events.emit('done', new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
 
             events.emit('done', null);
@@ -159,15 +160,18 @@ function testConfig(apiConfig, callback) {
 
     if (!apiConfig.backupFolder) return callback(new BackupsError(BackupsError.BAD_FIELD, 'backupFolder is required'));
 
-    fs.stat(apiConfig.backupFolder, function (error, result) {
-        if (error) {
-            debug('testConfig: %s', apiConfig.backupFolder, error);
-            return callback(new BackupsError(BackupsError.BAD_FIELD, 'Directory does not exist or cannot be accessed'));
-        }
+    if ('noHardlinks' in apiConfig && typeof apiConfig.noHardlinks !== 'boolean') return callback(new BackupsError(BackupsError.BAD_FIELD, 'noHardlinks must be boolean'));
 
+    fs.stat(apiConfig.backupFolder, function (error, result) {
+        if (error) return callback(new BackupsError(BackupsError.BAD_FIELD, 'Directory does not exist or cannot be accessed: ' + error.message));
         if (!result.isDirectory()) return callback(new BackupsError(BackupsError.BAD_FIELD, 'Backup location is not a directory'));
 
-        callback(null);
+        mkdirp(path.join(apiConfig.backupFolder, 'snapshot'), function (error) {
+            if (error && error.code === 'EACCES') return callback(new BackupsError(BackupsError.BAD_FIELD, `Access denied. Run "chown yellowtent:yellowtent ${apiConfig.backupFolder}" on the server`));
+            if (error) return callback(new BackupsError(BackupsError.BAD_FIELD, error.message));
+
+            callback(null);
+        });
     });
 }
 

src/storage/s3.js

@@ -108,7 +108,7 @@ function getS3Config(apiConfig, callback) {
 
     if (apiConfig.endpoint) credentials.endpoint = apiConfig.endpoint;
 
-    if (apiConfig.acceptSelfSignedCerts === true) {
+    if (apiConfig.acceptSelfSignedCerts === true && credentials.endpoint && credentials.endpoint.startsWith('https://')) {
         credentials.httpOptions.agent = {
             agent: new https.Agent({ rejectUnauthorized: false })
         };
@@ -190,7 +190,7 @@ function download(apiConfig, backupFilePath, callback) {
     });
 }
 
-function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callback) {
+function listDir(apiConfig, backupFilePath, iteratorCallback, callback) {
     getS3Config(apiConfig, function (error, credentials) {
         if (error) return callback(error);
 
@@ -207,10 +207,9 @@ function listDir(apiConfig, backupFilePath, batchSize, iteratorCallback, callbac
                     return foreverCallback(error);
                 }
 
-                var arr = batchSize === 1 ? listData.Contents : chunk(listData.Contents, batchSize);
-                if (arr.length === 0) return foreverCallback(new Error('Done'));
+                if (listData.Contents.length === 0) return foreverCallback(new Error('Done'));
 
-                iteratorCallback(s3, arr, function (error) {
+                iteratorCallback(s3, listData.Contents, function (error) {
                     if (error) return foreverCallback(error);
 
                     if (!listData.IsTruncated) return foreverCallback(new Error('Done'));
@@ -262,9 +261,9 @@ function downloadDir(apiConfig, backupFilePath, destDir) {
         });
     }
 
-    const concurrency = 10, batchSize = 1;
+    const concurrency = 10;
 
-    listDir(apiConfig, backupFilePath, batchSize, function (s3, objects, done) {
+    listDir(apiConfig, backupFilePath, function (s3, objects, done) {
         total += objects.length;
         async.eachLimit(objects, concurrency, downloadFile.bind(null, s3), done);
     }, function (error) {
@@ -370,10 +369,9 @@ function copy(apiConfig, oldFilePath, newFilePath) {
         });
     }
 
-    const batchSize = 1;
     var total = 0, concurrency = 4;
 
-    listDir(apiConfig, oldFilePath, batchSize, function (s3, objects, done) {
+    listDir(apiConfig, oldFilePath, function (s3, objects, done) {
         total += objects.length;
 
         if (retryCount === 0) concurrency = Math.min(concurrency + 1, 10); else concurrency = Math.max(concurrency - 1, 5);
@@ -429,7 +427,6 @@ function removeDir(apiConfig, pathPrefix) {
                 Objects: contents.map(function (c) { return { Key: c.Key }; })
             }
         };
-        total += contents.length;
 
         events.emit('progress', `Removing ${contents.length} files from ${contents[0].Key} to ${contents[contents.length-1].Key}`);
 
@@ -443,9 +440,14 @@ function removeDir(apiConfig, pathPrefix) {
         });
     }
 
-    const batchSize = apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; // throttle requests per second
+    listDir(apiConfig, pathPrefix, function (s3, objects, done) {
+        total += objects.length;
 
-    listDir(apiConfig, pathPrefix, batchSize, deleteFiles, function (error) {
+        const batchSize = apiConfig.provider !== 'digitalocean-spaces' ? 1000 : 100; // throttle objects in each request
+        var chunks = batchSize === 1 ? objects : chunk(objects, batchSize);
+
+        async.eachSeries(chunks, deleteFiles.bind(null, s3), done);
+    }, function (error) {
         events.emit('progress', `Removed ${total} files`);
 
         events.emit('done', error);

src/test/apps-test.js

@@ -135,7 +135,7 @@ describe('Apps', function () {
 
     describe('validateHostname', function () {
         it('does not allow admin subdomain', function () {
-            expect(apps._validateHostname(constants.ADMIN_LOCATION, 'cloudron.us')).to.be.an(Error);
+            expect(apps._validateHostname('my', 'cloudron.us')).to.be.an(Error);
         });
 
         it('cannot have >63 length subdomains', function () {

src/test/config-test.js

@@ -39,7 +39,7 @@ describe('config', function () {
     it('did set default values', function () {
         expect(config.isCustomDomain()).to.equal(true);
         expect(config.fqdn()).to.equal('localhost');
-        expect(config.adminOrigin()).to.equal('https://' + constants.ADMIN_LOCATION + '.localhost');
+        expect(config.adminOrigin()).to.equal('https://my.localhost');
         expect(config.appFqdn('app')).to.equal('app.localhost');
         expect(config.zoneName()).to.equal('');
     });
@@ -68,7 +68,7 @@ describe('config', function () {
 
         expect(config.isCustomDomain()).to.equal(true);
         expect(config.fqdn()).to.equal('example.com');
-        expect(config.adminOrigin()).to.equal('https://' + constants.ADMIN_LOCATION + '.example.com');
+        expect(config.adminOrigin()).to.equal('https://my.example.com');
         expect(config.appFqdn('app')).to.equal('app.example.com');
         expect(config.zoneName()).to.equal('example.com');
     });
@@ -79,7 +79,7 @@ describe('config', function () {
 
         expect(config.isCustomDomain()).to.equal(false);
         expect(config.fqdn()).to.equal('test.example.com');
-        expect(config.adminOrigin()).to.equal('https://' + constants.ADMIN_LOCATION + '-test.example.com');
+        expect(config.adminOrigin()).to.equal('https://my-test.example.com');
         expect(config.appFqdn('app')).to.equal('app-test.example.com');
         expect(config.zoneName()).to.equal('example.com');
     });

src/test/digest-test.js

@@ -16,6 +16,7 @@ var async = require('async'),
     paths = require('../paths.js'),
     safe = require('safetydance'),
     settings = require('../settings.js'),
+    settingsdb = require('../settingsdb.js'),
     updatechecker = require('../updatechecker.js'),
     user = require('../user.js');
 
@@ -113,7 +114,7 @@ describe('digest', function () {
             });
         });
 
-        it('sends mail for pending update to appstore account email (caas)', function (done) {
+        it('sends mail for pending update to owner account email', function (done) {
             var subscription = {
                 id: 'caas',
                 created: 0,
@@ -123,23 +124,16 @@ describe('digest', function () {
             };
 
             updatechecker._setUpdateInfo({ box: null, apps: { 'appid': { manifest: { version: '1.2.5', changelog: 'noop\nreally' } } } });
-            var fake1 = nock(config.apiServerOrigin()).post(function (uri) { return uri.indexOf('/api/v1/users/test-user/cloudrons') >= 0; }).reply(201, { cloudron: { id: 'test-cloudron' }});
-            var fake2 = nock(config.apiServerOrigin()).get(function (uri) { return uri.indexOf('/api/v1/users/test-user/cloudrons/test-cloudron/subscription') >= 0; }).reply(200, { subscription: subscription });
-            var fake3 = nock(config.apiServerOrigin()).get('/api/v1/users/test-user?accessToken=test-token').reply(200, { profile: { id: 'test-user', email: 'test@email.com' } });
 
-            settings.setAppstoreConfig({ userId: 'test-user', token: 'test-token', cloudronId: 'test-cloudron' }, function (error) {
+            settingsdb.set(settings.MAIL_CONFIG_KEY, JSON.stringify({ enabled: true }), function (error) {
                 if (error) return done(error);
 
                 digest.maybeSend(function (error) {
                     if (error) return done(error);
 
-                    checkMails(1, 'test@email.com', function (error) {
+                    checkMails(1, [ 'user0@email.com, username0@localhost' ], function (error) {
                         if (error) return done(error);
 
-                        expect(fake1.isDone()).to.be.ok();
-                        expect(fake2.isDone()).to.be.ok();
-                        expect(fake3.isDone()).to.be.ok();
-
                         done();
                     });
                 });

src/test/ldap-test.js

@@ -342,6 +342,35 @@ describe('Ldap', function () {
             });
         });
 
+        it ('succeeds with pagination', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: 'objectcategory=person',
+                paged: true
+            };
+
+            client.search('ou=users,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+                    entries.sort(function (a, b) { return a.username > b.username; });
+                    expect(entries[0].username).to.equal(USER_0.username.toLowerCase());
+                    expect(entries[0].mail).to.equal(USER_0.email.toLowerCase());
+                    expect(entries[1].username).to.equal(USER_1.username.toLowerCase());
+                    expect(entries[1].mail).to.equal(USER_1.email.toLowerCase());
+                    done();
+                });
+            });
+        });
+
         it ('succeeds with basic filter and email enabled', function (done) {
             // user settingsdb instead of settings, to not trigger further events
             settingsdb.set(settings.MAIL_CONFIG_KEY, JSON.stringify({ enabled: true }), function (error) {
@@ -617,13 +646,50 @@ describe('Ldap', function () {
                 });
             });
         });
+
+        it ('succeeds with pagination', function (done) {
+            var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
+
+            var opts = {
+                filter: 'objectclass=group',
+                paged: true
+            };
+
+            client.search('ou=groups,dc=cloudron', opts, function (error, result) {
+                expect(error).to.be(null);
+                expect(result).to.be.an(EventEmitter);
+
+                var entries = [];
+
+                result.on('searchEntry', function (entry) { entries.push(entry.object); });
+                result.on('error', done);
+                result.on('end', function (result) {
+                    expect(result.status).to.equal(0);
+                    expect(entries.length).to.equal(2);
+
+                    // ensure order for testability
+                    entries.sort(function (a, b) { return a.username < b.username; });
+
+                    expect(entries[0].cn).to.equal('users');
+                    expect(entries[0].memberuid.length).to.equal(3);
+                    expect(entries[0].memberuid[0]).to.equal(USER_0.id);
+                    expect(entries[0].memberuid[1]).to.equal(USER_1.id);
+                    expect(entries[0].memberuid[2]).to.equal(USER_2.id);
+                    expect(entries[1].cn).to.equal('admins');
+                    // if only one entry, the array becomes a string :-/
+                    expect(entries[1].memberuid).to.equal(USER_0.id);
+                    done();
+                });
+            });
+        });
     });
 
     function ldapSearch(dn, filter, callback) {
         var client = ldap.createClient({ url: 'ldap://127.0.0.1:' + config.get('ldapPort') });
 
         var opts = {
-            filter: filter
+            filter: filter,
+            paged: true
         };
 
         client.search(dn, opts, function (error, result) {

src/test/settings-test.js

@@ -162,29 +162,6 @@ describe('Settings', function () {
             });
         });
 
-        it('can get default update config config', function (done) {
-            settings.getUpdateConfig(function (error, updateConfig) {
-                expect(error).to.be(null);
-                expect(updateConfig.prerelease).to.be(false);
-                done();
-            });
-        });
-
-        it('can set update config', function (done) {
-            settings.setUpdateConfig({ prerelease: true }, function (error) {
-                expect(error).to.be(null);
-                done();
-            });
-        });
-
-        it('can get update config', function (done) {
-            settings.getUpdateConfig(function (error, updateConfig) {
-                expect(error).to.be(null);
-                expect(updateConfig.prerelease).to.be(true);
-                done();
-            });
-        });
-
         it('can set mail config', function (done) {
             settings.setMailConfig({ enabled: true }, function (error) {
                 expect(error).to.be(null);

src/test/updatechecker-test.js

@@ -66,7 +66,8 @@ describe('updatechecker - box - manual (email)', function () {
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settings.setAutoupdatePattern.bind(null, constants.AUTOUPDATE_PATTERN_NEVER),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' })),
-            mailer._clearMailQueue
+            mailer._clearMailQueue,
+            mailer.start
         ], done);
     });
 
@@ -113,7 +114,7 @@ describe('updatechecker - box - manual (email)', function () {
         });
     });
 
-    it('does not offer prerelease', function (done) {
+    it('offers prerelease', function (done) {
         nock.cleanAll();
 
         var scope = nock('http://localhost:4444')
@@ -121,39 +122,18 @@ describe('updatechecker - box - manual (email)', function () {
             .query({ boxVersion: config.version(), accessToken: 'token' })
             .reply(200, { version: '2.0.0-pre.0', changelog: [''], sourceTarballUrl: '2.0.0-pre.0.tar.gz' } );
 
+        var scope2 = nock('http://localhost:4444')
+            .get('/api/v1/users/uid/cloudrons/cid/subscription')
+            .query({ accessToken: 'token' })
+            .reply(200, { subscription: { plan: { id: 'pro' } } } );
+
         updatechecker.checkBoxUpdates(function (error) {
             expect(!error).to.be.ok();
-            expect(updatechecker.getUpdateInfo().box).to.be(null);
+            expect(updatechecker.getUpdateInfo().box.version).to.be('2.0.0-pre.0');
             expect(scope.isDone()).to.be.ok();
+            expect(scope2.isDone()).to.be.ok();
 
-            checkMails(0, done);
-        });
-    });
-
-    it('offers prerelease', function (done) {
-        nock.cleanAll();
-
-        settings.setUpdateConfig({ prerelease: true }, function (error) {
-            if (error) return done(error);
-
-            var scope = nock('http://localhost:4444')
-                .get('/api/v1/users/uid/cloudrons/cid/boxupdate')
-                .query({ boxVersion: config.version(), accessToken: 'token' })
-                .reply(200, { version: '2.0.0-pre.0', changelog: [''], sourceTarballUrl: '2.0.0-pre.0.tar.gz' } );
-
-            var scope2 = nock('http://localhost:4444')
-                .get('/api/v1/users/uid/cloudrons/cid/subscription')
-                .query({ accessToken: 'token' })
-                .reply(200, { subscription: { plan: { id: 'pro' } } } );
-
-            updatechecker.checkBoxUpdates(function (error) {
-                expect(!error).to.be.ok();
-                expect(updatechecker.getUpdateInfo().box.version).to.be('2.0.0-pre.0');
-                expect(scope.isDone()).to.be.ok();
-                expect(scope2.isDone()).to.be.ok();
-
-                checkMails(1, done);
-            });
+            checkMails(1, done);
         });
     });
 
@@ -184,6 +164,7 @@ describe('updatechecker - box - automatic (no email)', function () {
             database.initialize,
             settings.initialize,
             mailer._clearMailQueue,
+            mailer.start,
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE),
             settingsdb.set.bind(null, settings.APPSTORE_CONFIG_KEY, JSON.stringify({ userId: 'uid', cloudronId: 'cid', token: 'token' }))
         ], done);

src/updatechecker.js

@@ -19,7 +19,6 @@ var apps = require('./apps.js'),
     mailer = require('./mailer.js'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
-    semver = require('semver'),
     settings = require('./settings.js');
 
 var gAppUpdateInfo = { }, // id -> update info { creationDate, manifest }
@@ -114,7 +113,7 @@ function checkAppUpdates(callback) {
                     // always send notifications if user is on the free plan
                     if (result.plan.id === 'free' || result.plan.id === 'undecided') {
                         debug('Notifying user of app update for %s from %s to %s', app.id, app.manifest.version, updateInfo.manifest.version);
-                        mailer.appUpdateAvailable(app, updateInfo);
+                        mailer.appUpdateAvailable(app, false /* subscription */, updateInfo);
                         return iteratorDone();
                     }
 
@@ -124,7 +123,7 @@ function checkAppUpdates(callback) {
                             debug(error);
                         } else if (result === constants.AUTOUPDATE_PATTERN_NEVER) {
                             debug('Notifying user of app update for %s from %s to %s', app.id, app.manifest.version, updateInfo.manifest.version);
-                            mailer.appUpdateAvailable(app, updateInfo);
+                            mailer.appUpdateAvailable(app, true /* hasSubscription */, updateInfo);
                         }
 
                         iteratorDone();
@@ -149,48 +148,37 @@ function checkBoxUpdates(callback) {
     appstore.getBoxUpdate(function (error, updateInfo) {
         if (error || !updateInfo) return callback(error);
 
-        settings.getUpdateConfig(function (error, updateConfig) {
+        gBoxUpdateInfo = updateInfo;
+
+        // decide whether to send email
+        var state = loadState();
+
+        if (state.box === gBoxUpdateInfo.version) {
+            debug('Skipping notification of box update as user was already notified');
+            return callback();
+        }
+
+        appstore.getSubscription(function (error, result) {
             if (error) return callback(error);
 
-            var isPrerelease = semver.parse(updateInfo.version).prerelease.length !== 0;
-
-            if (isPrerelease && !updateConfig.prerelease) {
-                debug('Skipping update %s since this box does not want prereleases', updateInfo.version);
-                return callback();
+            function done() {
+                state.box = updateInfo.version;
+                saveState(state);
+                callback();
             }
 
-            gBoxUpdateInfo = updateInfo;
-
-            // decide whether to send email
-            var state = loadState();
-
-            if (state.box === gBoxUpdateInfo.version) {
-                debug('Skipping notification of box update as user was already notified');
-                return callback();
+            // always send notifications if user is on the free plan
+            if (result.plan.id === 'free' || result.plan.id === 'undecided') {
+                mailer.boxUpdateAvailable(false /* hasSubscription */, updateInfo.version, updateInfo.changelog);
+                return done();
             }
 
-            appstore.getSubscription(function (error, result) {
-                if (error) return callback(error);
+            // only send notifications if update pattern is 'never'
+            settings.getAutoupdatePattern(function (error, result) {
+                if (error) debug(error);
+                else if (result === constants.AUTOUPDATE_PATTERN_NEVER) mailer.boxUpdateAvailable(true /* hasSubscription */, updateInfo.version, updateInfo.changelog);
 
-                function done() {
-                    state.box = updateInfo.version;
-                    saveState(state);
-                    callback();
-                }
-
-                // always send notifications if user is on the free plan
-                if (result.plan.id === 'free' || result.plan.id === 'undecided') {
-                    mailer.boxUpdateAvailable(updateInfo.version, updateInfo.changelog);
-                    return done();
-                }
-
-                // only send notifications if update pattern is 'never'
-                settings.getAutoupdatePattern(function (error, result) {
-                    if (error) debug(error);
-                    else if (result === constants.AUTOUPDATE_PATTERN_NEVER) mailer.boxUpdateAvailable(updateInfo.version, updateInfo.changelog);
-
-                    done();
-                });
+                done();
             });
         });
     });

src/userdb.js

@@ -71,14 +71,14 @@ function getByEmail(email, callback) {
 function getOwner(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    // the first created user it the admin
+    // the first created user it the 'owner'
     database.query('SELECT ' + USERS_FIELDS + ' FROM users, groupMembers WHERE groupMembers.groupId = ? AND users.id = groupMembers.userId ORDER BY createdAt LIMIT 1',
-            [ constants.ADMIN_GROUP_ID ], function (error, result) {
-        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+        [ constants.ADMIN_GROUP_ID ], function (error, result) {
+            if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+            if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        callback(null, postProcess(result[0]));
-    });
+            callback(null, postProcess(result[0]));
+        });
 }
 
 function getByResetToken(resetToken, callback) {
@@ -116,7 +116,8 @@ function getAllWithGroupIds(callback) {
 function getAllAdmins(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + USERS_FIELDS + ' FROM users, groupMembers WHERE groupMembers.groupId = ? AND users.id = groupMembers.userId ORDER BY username', [ constants.ADMIN_GROUP_ID ], function (error, results) {
+    // the mailer code relies on the first object being the 'owner' (thus the ORDER)
+    database.query('SELECT ' + USERS_FIELDS + ' FROM users, groupMembers WHERE groupMembers.groupId = ? AND users.id = groupMembers.userId ORDER BY createdAt', [ constants.ADMIN_GROUP_ID ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         results.forEach(postProcess);

webadmin/src/js/client.js

@@ -366,10 +366,9 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
         }).error(defaultErrorHandler(callback));
     };
 
-    Client.prototype.updateApp = function (id, manifest, portBindings, callback) {
+    Client.prototype.updateApp = function (id, manifest, callback) {
         var data =  {
-            appStoreId: manifest.id + '@' + manifest.version,
-            portBindings: portBindings
+            appStoreId: manifest.id + '@' + manifest.version
         };
 
         post('/api/v1/apps/' + id + '/update', data).success(function (data, status) {
@@ -698,13 +697,6 @@ angular.module('Application').service('Client', ['$http', 'md5', 'Notification',
         }).error(defaultErrorHandler(callback));
     };
 
-    Client.prototype.getNonApprovedApps = function (callback) {
-        get('/api/v1/developer/apps').success(function (data, status) {
-            if (status !== 200 || typeof data !== 'object') return callback(new ClientError(status, data));
-            callback(null, data.apps || []);
-        }).error(defaultErrorHandler(callback));
-    };
-
     Client.prototype.getApp = function (appId, callback) {
         var appFound = null;
         this._installedApps.some(function (app) {

webadmin/src/views/apps.html

@@ -109,7 +109,7 @@
                         <a href="" ng-click="appConfigure.advancedVisible = true" ng-hide="appConfigure.advancedVisible">Advanced settings...</a>
                         <div  uib-collapse="!appConfigure.advancedVisible">
                             <div class="form-group">
-                                <label class="control-label" for="memoryLimit">Maximum Memory Limit: <b>{{ appConfigure.memoryLimit ? appConfigure.memoryLimit / 1024 / 1024 + 'MB' : 'Default (256 MB)' }}</b></label>
+                                <label class="control-label" for="memoryLimit">Maximum Memory Limit <sup><a ng-href="{{ config.webServerOrigin }}/documentation/apps/#increasing-the-memory-limit-of-an-app" class="help" target="_blank"><i class="fa fa-question-circle"></i></a></sup> : <b>{{ appConfigure.memoryLimit ? appConfigure.memoryLimit / 1024 / 1024 + 'MB' : 'Default (256 MB)' }}</b></label>
                                 <br/>
                                 <div style="padding: 0 10px;">
                                     <slider id="memoryLimit" ng-model="appConfigure.memoryLimit" step="134217728" tooltip="hide" ticks="appConfigure.memoryTicks" ticks-snap-bounds="67108864"></slider>
@@ -179,8 +179,11 @@
             <div class="modal-header">
                 <h4 class="modal-title">Restore {{ appRestore.app.fqdn }}</h4>
             </div>
-            <div class="modal-body" ng-show="appRestore.backups.length === 0">
-                <p class="text-danger">This app has no backups.</p>
+            <div class="modal-body" ng-show="appRestore.busyFetching">
+                <h4 class="text-center"><i class="fa fa-circle-o-notch fa-spin"></i> Fetching backups</h4>
+            </div>
+            <div class="modal-body" ng-show="appRestore.backups.length === 0 && !appRestore.busyFetching">
+                <h4 class="text-danger">This app has no backups.</h4>
             </div>
             <div class="modal-body" ng-show="appRestore.backups.length !== 0">
                 <p>Restoring the app will lose all content generated since the backup.</p>
@@ -195,7 +198,7 @@
                 </div>
                 <br/>
                 <fieldset>
-                    <form role="form" name="appRestoreForm" ng-submit="doRestore()" autocomplete="off">
+                    <form role="form" name="appRestoreForm" ng-submit="appRestore.submit()" autocomplete="off">
                         <div class="form-group" ng-class="{ 'has-error': (appRestoreForm.password.$dirty && appRestoreForm.password.$invalid) || (!appRestoreForm.password.$dirty && appRestore.error.password) }">
                             <label class="control-label" for="appRestorePasswordInput">Provide your password to confirm this action</label>
                             <div class="control-label" ng-show="(appRestoreForm.password.$dirty && appRestoreForm.password.$invalid) || (!appRestoreForm.password.$dirty && appRestore.error.password)">
@@ -211,7 +214,7 @@
             </div>
             <div class="modal-footer">
                 <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
-                <button type="button" class="btn btn-success" ng-click="doRestore()" ng-disabled="appRestoreForm.$invalid || appRestore.busy || !appRestore.selectedBackup"><i class="fa fa-circle-o-notch fa-spin" ng-show="appRestore.busy"></i> Restore</button>
+                <button type="button" class="btn btn-success" ng-click="appRestore.submit()" ng-show="appRestore.backups.length !== 0" ng-disabled="appRestoreForm.$invalid || appRestore.busy || !appRestore.selectedBackup"><i class="fa fa-circle-o-notch fa-spin" ng-show="appRestore.busy"></i> Restore</button>
             </div>
         </div>
     </div>
@@ -295,34 +298,11 @@
             </div>
             <div class="modal-body">
                 <p>Recent Changes for new version <b>{{ appUpdate.manifest.version}}</b>:</p>
-                <pre>{{ appUpdate.manifest.changelog }}</pre>
-                <br/>
-                <fieldset>
-                    <form role="form" name="appUpdateForm" ng-submit="doUpdate(appUpdateForm)" autocomplete="off">
-                        <div ng-repeat="(env, info) in appUpdate.portBindingsInfo" ng-class="{ 'newPort': info.isNew }">
-                            <ng-form name="portInfo_form">
-                                <div class="form-group" ng-class="{ 'has-error': portInfo_form.itemName{{$index}}.$dirty && portInfo_form.itemName{{$index}}.$invalid }">
-                                    <label class="control-label" for="inputPortInfo{{env}}"><input type="checkbox" ng-model="appUpdate.portBindingsEnabled[env]"> <span ng-show="info.isNew">New - </span> {{ info.description }} ({{ HOST_PORT_MIN }} - {{ HOST_PORT_MAX }})</label>
-                                    <input type="number" class="form-control" ng-model="appUpdate.portBindings[env]" ng-disabled="!appUpdate.portBindingsEnabled[env]" id="inputPortInfo{{env}}" later-name="itemName{{$index}}" min="{{HOST_PORT_MIN}}" max="{{HOST_PORT_MAX}}" required>
-                                </div>
-                            </ng-form>
-                        </div>
-                        <div ng-repeat="(env, port) in appUpdate.obsoletePortBindings" class="obsoletePort">
-                            <ng-form name="obsoletePortInfo_form">
-                                <div class="form-group">
-                                    Obsolete -
-                                    <label class="control-label">{{ env }}</label>
-                                    <input type="number" class="form-control" ng-model="port" disabled>
-                                </div>
-                            </ng-form>
-                        </div>
-                        <input class="ng-hide" type="submit" ng-disabled="appUpdateForm.$invalid || busy"/>
-                    </form>
-                </fieldset>
+                <div ng-bind-html="appUpdate.manifest.changelog | markdown2html"></div>
             </div>
             <div class="modal-footer">
                 <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
-                <button type="button" class="btn btn-danger" ng-click="doUpdate(appUpdateForm)" ng-disabled="appUpdateForm.$invalid || appUpdate.busy"><i class="fa fa-circle-o-notch fa-spin" ng-show="appUpdate.busy"></i> Update</button>
+                <button type="button" class="btn btn-danger" ng-click="doUpdate()" ng-disabled="appUpdate.busy"><i class="fa fa-circle-o-notch fa-spin" ng-show="appUpdate.busy"></i> Update</button>
             </div>
         </div>
     </div>
@@ -401,7 +381,7 @@
                     <div class="grid-item-bottom-mobile" ng-show="user.admin">
                         <div class="row">
                             <div class="col-xs-4 text-left">
-                                <a href="" ng-click="showRestore(app)" ng-show="backupConfig.provider !== 'noop'">
+                                <a href="" ng-click="appRestore.show(app)" ng-show="backupConfig.provider !== 'noop'">
                                     <i class="fa fa-undo scale"></i>
                                 </a>
 
@@ -424,7 +404,7 @@
                         </div>
 
                         <div>
-                            <a href="" ng-click="showRestore(app)" ng-show="backupConfig.provider !== 'noop'" title="Restore App"><i class="fa fa-undo scale"></i></a>
+                            <a href="" ng-click="appRestore.show(app)" ng-show="backupConfig.provider !== 'noop'" title="Restore App"><i class="fa fa-undo scale"></i></a>
                         </div>
 
                         <div ng-show="(app.installationState === 'installed' || app.installationState === 'pending_configure') && !(app | installError)">

webadmin/src/views/apps.js

@@ -63,6 +63,7 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
 
     $scope.appRestore = {
         busy: false,
+        busyFetching: false,
         error: {},
         app: {},
         password: '',
@@ -71,6 +72,47 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
 
         selectBackup: function (backup) {
             $scope.appRestore.selectedBackup = backup;
+        },
+
+        show: function (app) {
+            $scope.reset();
+
+            $scope.appRestore.app = app;
+            $scope.appRestore.busyFetching = true;
+
+            $('#appRestoreModal').modal('show');
+
+            Client.getAppBackups(app.id, function (error, backups) {
+                if (error) {
+                    Client.error(error);
+                } else {
+                    $scope.appRestore.backups = backups;
+                    if (backups.length) $scope.appRestore.selectedBackup = backups[0]; // pre-select first backup
+                    $scope.appRestore.busyFetching = false;
+                }
+            });
+
+            return false; // prevent propagation and default
+        },
+
+        submit: function () {
+            $scope.appRestore.busy = true;
+            $scope.appRestore.error.password = null;
+
+            Client.restoreApp($scope.appRestore.app.id, $scope.appRestore.selectedBackup.id, $scope.appRestore.password, function (error) {
+                if (error && error.statusCode === 403) {
+                    $scope.appRestore.password = '';
+                    $scope.appRestore.error.password = true;
+                    $scope.appRestoreForm.password.$setPristine();
+                    $('#appRestorePasswordInput').focus();
+                } else if (error) {
+                    Client.error(error);
+                } else {
+                    $('#appRestoreModal').modal('hide');
+                }
+
+                $scope.appRestore.busy = false;
+            });
         }
     };
 
@@ -136,10 +178,6 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         $scope.appUpdate.error = {};
         $scope.appUpdate.app = {};
         $scope.appUpdate.manifest = {};
-        $scope.appUpdate.portBindings = {};
-
-        $scope.appUpdateForm.$setPristine();
-        $scope.appUpdateForm.$setUntouched();
 
         // reset restore dialog
         $scope.appRestore.error = {};
@@ -214,14 +252,12 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         $scope.appConfigure.robotsTxt = app.robotsTxt;
         $scope.appConfigure.enableBackup = app.enableBackup;
 
-        // create ticks starting from manifest memory limit
-        $scope.appConfigure.memoryTicks = [
-             256 * 1024 * 1024,
-             512 * 1024 * 1024,
-             1024 * 1024 * 1024,
-             2048 * 1024 * 1024,
-             4096 * 1024 * 1024
-        ].filter(function (t) { return t >= (app.manifest.memoryLimit || 0); });
+        // create ticks starting from manifest memory limit. the memory limit here is currently split into ram+swap (and thus *2 below)
+        // TODO: the *2 will overallocate since 4GB is max swap that cloudron itself allocates
+        $scope.appConfigure.memoryTicks = [ ];
+        for (var i = 256; i <= ($scope.config.memory*2/1024/1024); i *= 2) {
+            if (i >= (app.manifest.memoryLimit/1024/1024 || 0)) $scope.appConfigure.memoryTicks.push(i * 1024 * 1024);
+        }
         if (app.manifest.memoryLimit && $scope.appConfigure.memoryTicks[0] !== app.manifest.memoryLimit) {
             $scope.appConfigure.memoryTicks.unshift(app.manifest.memoryLimit);
         }
@@ -324,48 +360,6 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         return false; // prevent propagation and default
     };
 
-    $scope.showRestore = function (app) {
-        $scope.reset();
-
-        $scope.appRestore.app = app;
-        $scope.appRestore.busy = true;
-
-        $('#appRestoreModal').modal('show');
-
-        Client.getAppBackups(app.id, function (error, backups) {
-            if (error) {
-                Client.error(error);
-            } else {
-                $scope.appRestore.backups = backups;
-                if (backups.length) $scope.appRestore.selectedBackup = backups[0]; // pre-select first backup
-                $scope.appRestore.busy = false;
-            }
-        });
-
-        return false; // prevent propagation and default
-    };
-
-    $scope.doRestore = function () {
-        $scope.appRestore.busy = true;
-        $scope.appRestore.error.password = null;
-
-        Client.restoreApp($scope.appRestore.app.id, $scope.appRestore.selectedBackup.id, $scope.appRestore.password, function (error) {
-            if (error && error.statusCode === 403) {
-                $scope.appRestore.password = '';
-                $scope.appRestore.error.password = true;
-                $scope.appRestoreForm.password.$setPristine();
-                $('#appRestorePasswordInput').focus();
-            } else if (error) {
-                Client.error(error);
-            } else {
-                $('#appRestoreModal').modal('hide');
-                $scope.reset();
-            }
-
-            $scope.appRestore.busy = false;
-        });
-    };
-
     $scope.showUninstall = function (app) {
         $scope.reset();
 
@@ -408,90 +402,17 @@ angular.module('Application').controller('AppsController', ['$scope', '$location
         $scope.appUpdate.app = app;
         $scope.appUpdate.manifest = angular.copy(updateManifest);
 
-        // ensure we always operate on objects here
-        app.portBindings = app.portBindings || {};
-        app.manifest.tcpPorts = app.manifest.tcpPorts || {};
-        updateManifest.tcpPorts = updateManifest.tcpPorts || {};
-
-        // Activate below two lines for testing the UI
-        // updateManifest.tcpPorts['TEST_HTTP'] = { defaultValue: 1337, description: 'HTTP server'};
-        // app.manifest.tcpPorts['TEST_FOOBAR'] = { defaultValue: 1338, description: 'FOOBAR server'};
-        // app.portBindings['TEST_SSH'] = 1339;
-
-        var portBindingsInfo = {};                  // Portbinding map only for information
-        var portBindings = {};                      // This is the actual model holding the env:port pair
-        var portBindingsEnabled = {};               // This is the actual model holding the enabled/disabled flag
-        var obsoletePortBindings = {};              // Info map for obsolete port bindings, this is for display use only and thus not in the model
-        var portsChanged = false;
-        var env;
-
-        // detect new portbindings and copy all from manifest.tcpPorts
-        for (env in updateManifest.tcpPorts) {
-            portBindingsInfo[env] = updateManifest.tcpPorts[env];
-            if (!app.manifest.tcpPorts[env]) {
-                portBindingsInfo[env].isNew = true;
-                portBindingsEnabled[env] = true;
-
-                // use default integer port value in model
-                portBindings[env] = updateManifest.tcpPorts[env].defaultValue || 0;
-
-                portsChanged = true;
-            } else {
-                // detect if the port binding was enabled
-                if (app.portBindings[env]) {
-                    portBindings[env] = app.portBindings[env];
-                    portBindingsEnabled[env] = true;
-                } else {
-                    portBindings[env] = updateManifest.tcpPorts[env].defaultValue || 0;
-                    portBindingsEnabled[env] = false;
-                }
-            }
-        }
-
-        // detect obsolete portbindings (mappings in app.portBindings, but not anymore in updateManifest.tcpPorts)
-        for (env in app.manifest.tcpPorts) {
-            // only list the port if it is not in the new manifest and was enabled previously
-            if (!updateManifest.tcpPorts[env] && app.portBindings[env]) {
-                obsoletePortBindings[env] = app.portBindings[env];
-                portsChanged = true;
-            }
-        }
-
-        // now inject the maps into the $scope, we only show those if ports have changed
-        $scope.appUpdate.portBindings = portBindings;                 // always inject the model, so it gets used in the actual update call
-        $scope.appUpdate.portBindingsEnabled = portBindingsEnabled;   // always inject the model, so it gets used in the actual update call
-
-        if (portsChanged) {
-            $scope.appUpdate.portBindingsInfo = portBindingsInfo;
-            $scope.appUpdate.obsoletePortBindings = obsoletePortBindings;
-        } else {
-            $scope.appUpdate.portBindingsInfo = {};
-            $scope.appUpdate.obsoletePortBindings = {};
-        }
-
         $('#appUpdateModal').modal('show');
     };
 
-    $scope.doUpdate = function (form) {
+    $scope.doUpdate = function () {
         $scope.appUpdate.busy = true;
 
-        // only use enabled ports from portBindings
-        var finalPortBindings = {};
-        for (var env in $scope.appUpdate.portBindings) {
-            if ($scope.appUpdate.portBindingsEnabled[env]) {
-                finalPortBindings[env] = $scope.appUpdate.portBindings[env];
-            }
-        }
-
-        Client.updateApp($scope.appUpdate.app.id, $scope.appUpdate.manifest, finalPortBindings, function (error) {
+        Client.updateApp($scope.appUpdate.app.id, $scope.appUpdate.manifest, function (error) {
             if (error) {
                 Client.error(error);
             } else {
                 $scope.appUpdate.app = {};
-
-                form.$setPristine();
-                form.$setUntouched();
-
                 $('#appUpdateModal').modal('hide');
             }
 

webadmin/src/views/appstore.js

@@ -353,18 +353,6 @@ angular.module('Application').controller('AppStoreController', ['$scope', '$loca
             });
 
             return callback(null, apps);
-
-            // Client.getNonApprovedApps(function (error, result) {
-            //     if (error) return callback(error);
-
-            //     // add testing tag to the manifest for UI and search reasons
-            //     result.forEach(function (app) {
-            //         if (!app.manifest.tags) app.manifest.tags = [];
-            //         app.manifest.tags.push('testing');
-            //     });
-
-            //     callback(null, apps.concat(result));
-            // });
         });
     }
 

webadmin/src/views/certs.html

@@ -33,7 +33,7 @@
                         <div class="form-group" ng-class="{ 'has-error': false }" ng-show="dnsCredentials.provider === 'gcdns'">
                             <div class="input-group">
                                 <input type="file" id="gcdnsKeyFileInput" style="display:none"/>
-                                <input type="text" class="form-control" placeholder="Service Account Key" ng-model="dnsCredentials.gcdnsKey.keyFileName" id="gcdnsKeyInput" name="cert" onclick="getElementById('gcdnsKeyFileInput').click();" style="cursor: pointer;" ng-disabled="dnsCredentials.busy" required>
+                                <input type="text" class="form-control" placeholder="Service Account Key" ng-model="dnsCredentials.gcdnsKey.keyFileName" id="gcdnsKeyInput" name="cert" onclick="getElementById('gcdnsKeyFileInput').click();" style="cursor: pointer;" ng-disabled="dnsCredentials.busy" ng-required="dnsCredentials.provider === 'gcdns'">
                                 <span class="input-group-addon">
                                   <i class="fa fa-upload" onclick="getElementById('gcdnsKeyFileInput').click();"></i>
                               </span>
@@ -90,7 +90,7 @@
                 </p>
 
                 <p ng-show="dnsCredentials.provider === 'manual'">
-                    Setup an <i>A</i> record for <b>my.{{ dnsCredentials.customDomain || 'example.com' }}</b> to this server's IP. All DNS records have to be setup manually <i>before</i> each app installation.
+                    Setup an <i>A</i> record for <b>{{ config.adminLocation }}.{{ dnsCredentials.customDomain || 'example.com' }}</b> to this server's IP. All DNS records have to be setup manually <i>before</i> each app installation.
                 </p>
             </div>
             <div class="modal-footer ">

webadmin/src/views/email.html

@@ -65,9 +65,9 @@
        <a href="" data-toggle="collapse" data-parent="#accordion" data-target="#mail_settings">Mail server settings for email clients</a>
        <div id="mail_settings" class="panel-collapse collapse">
          <br/>
-         <p><b>Incoming Mail (IMAP)</b><br/>Server: <span ng-click-select>my.{{config.fqdn}}</span><br/>Port: 993 (TLS)</p>
-         <p><b>Outgoing Mail (SMTP)</b><br/>Server: <span ng-click-select>my.{{config.fqdn}}</span><br/>Port: 587 (STARTTLS)</p>
-         <p><b>ManageSieve</b><br/>Server: <span ng-click-select>my.{{config.fqdn}}</span><br/>Port: 4190 (TLS)</p>
+         <p><b>Incoming Mail (IMAP)</b><br/>Server: <span ng-click-select>{{config.mailFqdn}}</span><br/>Port: 993 (TLS)</p>
+         <p><b>Outgoing Mail (SMTP)</b><br/>Server: <span ng-click-select>{{config.mailFqdn}}</span><br/>Port: 587 (STARTTLS)</p>
+         <p><b>ManageSieve</b><br/>Server: <span ng-click-select>{{config.mailFqdn}}</span><br/>Port: 4190 (TLS)</p>
          <p>All the servers require your Cloudron credentials for authentication.</p>
         </div>
       </div>

webadmin/src/views/settings.html

@@ -146,6 +146,14 @@
                             </p>
                         </div>
 
+                        <div class="checkbox" ng-show="configureBackup.provider === 'filesystem'">
+                            <label>
+                                <input type="checkbox" ng-model="configureBackup.useHardlinks" id="inputConfigureUseHardlinks">
+                                    Use hardlinks
+                                </input>
+                            </label>
+                        </div>
+
                         <!-- S3/Minio/SOS -->
                         <div class="form-group" ng-class="{ 'has-error': configureBackup.error.endpoint }" ng-show="configureBackup.provider === 'minio' || configureBackup.provider === 's3-v4-compat'">
                             <label class="control-label" for="inputConfigureBackupEndpoint">Endpoint</label>

webadmin/src/views/settings.js

@@ -327,6 +327,7 @@ angular.module('Application').controller('SettingsController', ['$scope', '$loca
         backupFolder: '',
         retentionSecs:  7 * 24 * 60 * 60,
         acceptSelfSignedCerts: false,
+        useHardlinks: true,
         format: 'tgz',
 
         clearForm: function () {
@@ -340,6 +341,7 @@ angular.module('Application').controller('SettingsController', ['$scope', '$loca
             $scope.configureBackup.retentionSecs = 7 * 24 * 60 * 60;
             $scope.configureBackup.format = 'tgz';
             $scope.configureBackup.acceptSelfSignedCerts = false;
+            $scope.configureBackup.useHardlinks = true;
         },
 
         show: function () {
@@ -358,6 +360,7 @@ angular.module('Application').controller('SettingsController', ['$scope', '$loca
             $scope.configureBackup.retentionSecs = $scope.backupConfig.retentionSecs;
             $scope.configureBackup.format = $scope.backupConfig.format;
             $scope.configureBackup.acceptSelfSignedCerts = !!$scope.backupConfig.acceptSelfSignedCerts;
+            $scope.configureBackup.useHardlinks = !$scope.backupConfig.noHardlinks;
 
             $('#configureBackupModal').modal('show');
         },
@@ -397,6 +400,7 @@ angular.module('Application').controller('SettingsController', ['$scope', '$loca
                 }
             } else if (backupConfig.provider === 'filesystem') {
                 backupConfig.backupFolder = $scope.configureBackup.backupFolder;
+                backupConfig.noHardlinks = !$scope.configureBackup.useHardlinks;
             }
 
             Client.setBackupConfig(backupConfig, function (error) {