Do not set hostname of app containers

do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
name to look up the internal docker ip. this makes curl from within container fail
Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
Hostname cannot be set with container NetworkMode

.jshintrc

@@ -1,7 +1,8 @@
 {
-	"node": true,
-	"browser": true,
-	"unused": true,
-	"globalstrict": true,
-	"predef": [ "angular", "$" ]
+    "node": true,
+    "browser": true,
+    "unused": true,
+    "globalstrict": true,
+    "predef": [ "angular", "$" ],
+    "esnext": true
 }

CHANGES

@@ -507,3 +507,38 @@
 - Add search for activity log
 - Add tutorial for first time users
 
+[0.13.4]
+- Fix mail addon restart issue
+
+[0.14.0]
+- You have mail :-)
+
+[0.14.1]
+- 2-character usernames are now allowed
+- Make cloudron CLI push/pull more robust
+
+[0.14.2]
+- Update mail addon
+
+[0.15.0]
+- [REST API](https://cloudron.io/references/api.html) is now in public beta
+- Enable Developer mode by default for new Cloudrons
+- Reverse proxy fixes for apps exposing a WebDav server
+- Allow admins to optionally set the username and displayName on user creation
+- Fix app autoupdate logic to detect if one or more in-use port bindings was removed
+
+[0.15.1]
+- Fix mail connectivity from IPv6 clients
+- Add API token management UI
+- Improved UI to enter email aliases
+
+[0.15.2]
+- Allow restoring apps from any previous backup
+
+[0.15.3]
+- Show installation progress in a tooltip
+
+[0.16.0]
+- Allow apps to be configured in configuring state
+- Improved platform architecture that allows incremental infrastructure updates
+

baseimage/createImage

@@ -137,8 +137,8 @@ while true; do
     sleep 30
 done
 
-echo "Copying INFRA_VERSION"
-$scp22 "${SCRIPT_DIR}/../src/INFRA_VERSION" root@${server_ip}:.
+echo "Copying infra_version.js"
+$scp22 "${SCRIPT_DIR}/../src/infra_version.js" root@${server_ip}:.
 
 echo "Copying box source"
 cd "${SOURCE_DIR}"

baseimage/digitalocean.sh

@@ -10,7 +10,7 @@ if [[ -z "${JSON}" ]]; then
     exit 1
 fi
 
-readonly CURL="curl -s -u ${DIGITAL_OCEAN_TOKEN}:"
+readonly CURL="curl --retry 5 -s -u ${DIGITAL_OCEAN_TOKEN}:"
 
 function debug() {
     echo "$@" >&2
@@ -49,7 +49,7 @@ function get_droplet_ip() {
 
 function get_droplet_id() {
     local droplet_name="$1"
-    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=100" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
+    id=$($CURL "https://api.digitalocean.com/v2/droplets?per_page=200" | $JSON "droplets" | $JSON -c "this.name === '${droplet_name}'" | $JSON "[0].id")
     [[ -z "$id" ]] && exit 1
     echo "$id"  
 }
@@ -109,13 +109,24 @@ function get_image_id() {
     local snapshot_name="$1"
     local image_id=""
 
-    image_id=$($CURL "https://api.digitalocean.com/v2/images?per_page=100" \
-       | $JSON images \
-       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id)
-
-    if [[ -n "${image_id}" ]]; then
-        echo "${image_id}"
+    if ! response=$($CURL "https://api.digitalocean.com/v2/images?per_page=200"); then
+        echo "Failed to get image listing. ${response}"
+        return 1
     fi
+
+    if ! image_id=$(echo "$response" \
+       | $JSON images \
+       | $JSON -c "this.name === \"${snapshot_name}\"" 0.id); then
+        echo "Failed to parse curl response: ${response}"
+        return 1
+    fi
+
+    if [[ -z "${image_id}" ]]; then
+        echo "Failed to get image id of ${snapshot_name}. reponse: ${response}"
+        return 1
+    fi
+
+    echo "${image_id}"
 }
 
 function snapshot_droplet() {
@@ -128,16 +139,26 @@ function snapshot_droplet() {
     debug -n "Waiting for snapshot to complete"
 
     while true; do
-        local event_status=`$CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}" | $JSON action.status`
+        if ! response=$($CURL "https://api.digitalocean.com/v2/droplets/${droplet_id}/actions/${event_id}"); then
+            echo "Could not get action status. ${response}"
+            continue
+        fi
+        if ! event_status=$(echo "${response}" | $JSON action.status); then
+            echo "Could not parse action.status from response. ${response}"
+            continue
+        fi
         if [[ "${event_status}" == "completed" ]]; then
             break
         fi
         debug -n "."
         sleep 10
     done
-    debug ""
+    debug "! done"
 
-    get_image_id "${snapshot_name}"
+    if ! image_id=$(get_image_id "${snapshot_name}"); then
+        return 1
+    fi
+    echo "${image_id}"
 }
 
 function destroy_droplet() {

baseimage/initializeBaseUbuntuImage.sh

@@ -19,12 +19,6 @@ function die {
 
 [[ "$(systemd --version 2>&1)" == *"systemd 229"* ]] || die "Expecting systemd to be 229"
 
-if [ -f "${SOURCE_DIR}/INFRA_VERSION" ]; then
-    source "${SOURCE_DIR}/INFRA_VERSION"
-else
-    echo "No INFRA_VERSION found, skip pulling docker images"
-fi
-
 if [ ${SELFHOSTED} == 0 ]; then
     echo "!! Initializing Ubuntu image for CaaS"
 else
@@ -67,14 +61,14 @@ iptables -P OUTPUT ACCEPT
 # allow ssh, http, https, ping, dns
 iptables -I INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
 if [ ${SELFHOSTED} == 0 ]; then
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,202,443,886 -j ACCEPT
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,202,443,587,993,4190 -j ACCEPT
 else
-    iptables -A INPUT -p tcp -m tcp -m multiport --dports 80,22,443,886 -j ACCEPT
+    iptables -A INPUT -p tcp -m tcp -m multiport --dports 25,80,22,443,587,993,4190 -j ACCEPT
 fi
 iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT
 iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT
 iptables -A INPUT -p udp --sport 53 -j ACCEPT
-iptables -A INPUT -s 172.17.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
+iptables -A INPUT -s 172.18.0.0/16 -j ACCEPT # required to accept any connections from apps to our IP:<public port>
 
 # loopback
 iptables -A INPUT -i lo -j ACCEPT
@@ -156,30 +150,22 @@ update-grub
 # now add the user to the docker group
 usermod "${USER}" -a -G docker
 
-if [ -z $(echo "${INFRA_VERSION}") ]; then
-    echo "Skip pulling base docker images"
-else
-    echo "=== Pulling base docker images ==="
-    docker pull "${BASE_IMAGE}"
+echo "==== Install nodejs ===="
+# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
+mkdir -p /usr/local/node-4.1.1
+curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
+ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
+ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
+apt-get install -y python   # Install python which is required for npm rebuild
+[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
 
-    echo "=== Pulling mysql addon image ==="
-    docker pull "${MYSQL_IMAGE}"
+echo "==== Downloading docker images ===="
+images=$(node -e "var i = require('${SOURCE_DIR}/infra_version.js'); console.log(i.baseImage, Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 
-    echo "=== Pulling postgresql addon image ==="
-    docker pull "${POSTGRESQL_IMAGE}"
-
-    echo "=== Pulling redis addon image ==="
-    docker pull "${REDIS_IMAGE}"
-
-    echo "=== Pulling mongodb addon image ==="
-    docker pull "${MONGODB_IMAGE}"
-
-    echo "=== Pulling graphite docker images ==="
-    docker pull "${GRAPHITE_IMAGE}"
-
-    echo "=== Pulling mail relay ==="
-    docker pull "${MAIL_IMAGE}"
-fi
+echo "Pulling images: ${images}"
+for image in ${images}; do
+    docker pull "${image}"
+done
 
 echo "==== Install nginx ===="
 apt-get -y install nginx-full
@@ -210,15 +196,6 @@ echo "==== Install logrotate ==="
 apt-get install -y cron logrotate
 systemctl enable cron
 
-echo "==== Install nodejs ===="
-# Cannot use anything above 4.1.1 - https://github.com/nodejs/node/issues/3803
-mkdir -p /usr/local/node-4.1.1
-curl -sL https://nodejs.org/dist/v4.1.1/node-v4.1.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-4.1.1
-ln -s /usr/local/node-4.1.1/bin/node /usr/bin/node
-ln -s /usr/local/node-4.1.1/bin/npm /usr/bin/npm
-apt-get install -y python   # Install python which is required for npm rebuild
-[[ "$(python --version 2>&1)" == "Python 2.7."* ]] || die "Expecting python version to be 2.7.x"
-
 echo "=== Rebuilding npm packages ==="
 cd "${INSTALLER_SOURCE_DIR}" && npm install --production
 chown "${USER}:${USER}" -R "${INSTALLER_SOURCE_DIR}"

migrations/20160527010012-mailboxes-add-table.js

@@ -0,0 +1,24 @@
+'use strict';
+
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+	var cmd = 'CREATE TABLE mailboxes(' +
+				'name VARCHAR(128) NOT NULL,' +
+				'aliasTarget VARCHAR(128),' +
+				'creationTime TIMESTAMP,' +
+				'PRIMARY KEY (name))';
+
+    db.runSql(cmd, function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('DROP TABLE mailboxes', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/20160528090040-mailboxes-import-existing-users.js

@@ -0,0 +1,25 @@
+var dbm = global.dbm || require('db-migrate');
+var type = dbm.dataType;
+
+// imports mailbox entries for existing users
+exports.up = function(db, callback) {
+    async.series([
+        db.runSql.bind(db, 'START TRANSACTION;'),
+        function addUserMailboxes(done) {
+            db.all('SELECT username FROM users', function (error, results) {
+                if (error) return done(error);
+
+                async.eachSeries(results, function (r, next) {
+                    if (!r.username) return next();
+
+                    db.runSql('INSERT INTO mailboxes (name) VALUES (?)', [ r.username ], next);
+                }, done);
+            });
+        },
+       db.runSql.bind(db, 'COMMIT')
+    ], callback);
+};
+
+exports.down = function(db, callback) {
+  callback();
+};

migrations/20160614021819-apps-drop-lastBackupConfigJson.js

@@ -0,0 +1,16 @@
+dbm = dbm || require('db-migrate');
+var type = dbm.dataType;
+
+exports.up = function(db, callback) {
+    db.runSql('ALTER TABLE apps DROP COLUMN lastBackupConfigJson', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};
+
+exports.down = function(db, callback) {
+    db.runSql('ALTER TABLE apps ADD COLUMN lastBackupConfigJson TEXT', function (error) {
+        if (error) console.error(error);
+        callback(error);
+    });
+};

migrations/schema.sql

@@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS tokens(
     identifier VARCHAR(128) NOT NULL,
     clientId VARCHAR(128),
     scope VARCHAR(512) NOT NULL,
-    expires BIGINT NOT NULL,
+    expires BIGINT NOT NULL, // FIXME: make this a timestamp
     PRIMARY KEY(accessToken));
 
 CREATE TABLE IF NOT EXISTS clients(
@@ -62,16 +62,15 @@ CREATE TABLE IF NOT EXISTS apps(
     httpPort INTEGER,                        // this is the nginx proxy port and not manifest.httpPort
     location VARCHAR(128) NOT NULL UNIQUE,
     dnsRecordId VARCHAR(512),
-    accessRestrictionJson TEXT,
+    accessRestrictionJson TEXT, // { users: [ ], groups: [ ] }
     oauthProxy BOOLEAN DEFAULT 0,
     createdAt TIMESTAMP(2) NOT NULL DEFAULT CURRENT_TIMESTAMP,
     memoryLimit BIGINT DEFAULT 0,
     altDomain VARCHAR(256),
 
-    lastBackupId VARCHAR(128),
-    lastBackupConfigJson TEXT, // used for appstore and non-appstore installs. it's here so it's easy to do REST validation
+    lastBackupId VARCHAR(128), // tracks last valid backup, can be removed
 
-    oldConfigJson TEXT, // used to pass old config for apptask
+    oldConfigJson TEXT, // used to pass old config for apptask, can be removed when we use a queue
 
     PRIMARY KEY(id));
 
@@ -86,7 +85,7 @@ CREATE TABLE IF NOT EXISTS authcodes(
     authCode VARCHAR(128) NOT NULL UNIQUE,
     userId VARCHAR(128) NOT NULL,
     clientId VARCHAR(128) NOT NULL,
-    expiresAt BIGINT NOT NULL,
+    expiresAt BIGINT NOT NULL, // ## FIXME: make this a timestamp
     PRIMARY KEY(authCode));
 
 CREATE TABLE IF NOT EXISTS settings(
@@ -115,6 +114,17 @@ CREATE TABLE IF NOT EXISTS eventlog(
     action VARCHAR(128) NOT NULL,
     source JSON, /* { userId, username, ip }. userId can be null for cron,sysadmin */
     data JSON, /* free flowing json based on action */
+    creationTime TIMESTAMP, /* FIXME: precision must be TIMESTAMP(2) */
+
+    PRIMARY KEY (id));
+
+/* Future fields:
+   * accessRestriction - to determine who can access it. So this has foreign keys
+   * quota - per mailbox quota
+*/
+CREATE TABLE IF NOT EXISTS mailboxes(
+    name VARCHAR(128) NOT NULL,
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
     creationTime TIMESTAMP,
 
     PRIMARY KEY (id));

package.json

@@ -16,10 +16,9 @@
     "async": "^1.2.1",
     "aws-sdk": "^2.1.46",
     "body-parser": "^1.13.1",
-    "bytes": "^2.3.0",
-    "cloudron-manifestformat": "^2.3.0",
+    "cloudron-manifestformat": "^2.4.0",
     "connect-ensure-login": "^0.1.1",
-    "connect-lastmile": "0.0.13",
+    "connect-lastmile": "^0.1.0",
     "connect-timeout": "^1.5.0",
     "cookie-parser": "^1.3.5",
     "cookie-session": "^1.1.0",
@@ -33,6 +32,7 @@
     "express": "^4.12.4",
     "express-session": "^1.11.3",
     "hat": "0.0.3",
+    "ini": "^1.3.4",
     "json": "^9.0.3",
     "ldapjs": "^0.7.1",
     "mime": "^1.3.4",
@@ -56,7 +56,6 @@
     "proxy-middleware": "^0.13.0",
     "safetydance": "^0.1.1",
     "semver": "^4.3.6",
-    "serve-favicon": "^2.2.0",
     "split": "^1.0.0",
     "superagent": "^1.8.3",
     "supererror": "^0.7.1",
@@ -89,7 +88,6 @@
     "mocha": "*",
     "nock": "^3.4.0",
     "node-sass": "^3.0.0-alpha.0",
-    "redis": "^2.4.2",
     "request": "^2.65.0",
     "sinon": "^1.12.2",
     "yargs": "^3.15.0"

setup/container/sudoers

@@ -31,6 +31,3 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/collectlogs.sh
 Defaults!/home/yellowtent/box/src/scripts/retire.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/retire.sh
 
-Defaults!/home/yellowtent/box/src/scripts/setup_infra.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/setup_infra.sh
-

setup/splashpage.sh

@@ -9,8 +9,6 @@ readonly BOX_SRC_DIR="/home/yellowtent/box"
 readonly DATA_DIR="/home/yellowtent/data"
 readonly ADMIN_LOCATION="my" # keep this in sync with constants.js
 
-source "${script_dir}/../src/INFRA_VERSION" # this injects INFRA_VERSION
-
 echo "Setting up nginx update page"
 
 source "${script_dir}/argparser.sh" "$@" # this injects the arg_* variables used below
@@ -24,13 +22,16 @@ rm -rf "${SETUP_WEBSITE_DIR}" && mkdir -p "${SETUP_WEBSITE_DIR}"
 cp -r "${script_dir}/splash/website/"* "${SETUP_WEBSITE_DIR}"
 
 # create nginx config
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${arg_retire}" == "true" || "${infra_version}" != "${INFRA_VERSION}" ]]; then
+readonly current_infra=$(node -e "console.log(require('${script_dir}/../src/infra_version.js').version);")
+existing_infra="none"
+[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && existing_infra=$(node -e "console.log(JSON.parse(require('fs').readFileSync('${DATA_DIR}/INFRA_VERSION', 'utf8')).version);")
+if [[ "${arg_retire}" == "true" || "${existing_infra}" != "${current_infra}" ]]; then
+    echo "Showing progress bar on all subdomains in retired mode or infra update. retire: ${arg_retire} existing: ${existing_infra} current: ${current_infra}"
     rm -f ${DATA_DIR}/nginx/applications/*
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
         -O "{ \"vhost\": \"~^(.+)\$\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 else
+    echo "Show progress bar only on admin domain for normal update"
     ${BOX_SRC_DIR}/node_modules/.bin/ejs-cli -f "${script_dir}/start/nginx/appconfig.ejs" \
         -O "{ \"vhost\": \"${admin_fqdn}\", \"adminOrigin\": \"${admin_origin}\", \"endpoint\": \"splash\", \"sourceDir\": \"${SETUP_WEBSITE_DIR}\", \"certFilePath\": \"cert/host.cert\", \"keyFilePath\": \"cert/host.key\" }" > "${DATA_DIR}/nginx/applications/admin.conf"
 fi

setup/start.sh

@@ -120,6 +120,7 @@ fi
 
 set_progress "33" "Changing ownership"
 chown "${USER}:${USER}" -R "${DATA_DIR}/box" "${DATA_DIR}/nginx" "${DATA_DIR}/collectd" "${DATA_DIR}/addons" "${DATA_DIR}/acme"
+chown "${USER}:${USER}" "${DATA_DIR}/INFRA_VERSION" || true
 chown "${USER}:${USER}" "${DATA_DIR}"
 
 set_progress "65" "Creating cloudron.conf"
@@ -135,7 +136,6 @@ cat > "${CONFIG_DIR}/cloudron.conf" <<CONF_END
     "fqdn": "${arg_fqdn}",
     "isCustomDomain": ${arg_is_custom_domain},
     "boxVersionsUrl": "${arg_box_versions_url}",
-    "adminEmail": "\"Cloudron\" <no-reply@${arg_fqdn}>",
     "provider": "${arg_provider}",
     "database": {
         "hostname": "localhost",
@@ -188,18 +188,20 @@ if [[ ! -z "${arg_tls_config}" ]]; then
         -e "REPLACE INTO settings (name, value) VALUES (\"tls_config\", '$arg_tls_config')" box
 fi
 
-# Add webadmin oauth client
 # The domain might have changed, therefor we have to update the record
 # !!! This needs to be in sync with the webadmin, specifically login_callback.js
-echo "Add webadmin oauth cient"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add webadmin api cient"
+readonly ADMIN_SCOPES="cloudron,developer,profile,users,apps,settings"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"Settings\", \"built-in\", \"secret-webadmin\", \"${admin_origin}\", \"${ADMIN_SCOPES}\")" box
 
-echo "Add localhost test oauth client"
-ADMIN_SCOPES="root,developer,profile,users,apps,settings"
+echo "Add SDK api client"
 mysql -u root -p${mysql_root_password} \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-test\", \"test\", \"test\", \"secret-test\", \"http://127.0.0.1:5000\", \"${ADMIN_SCOPES}\")" box
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-sdk\", \"SDK\", \"built-in\", \"secret-sdk\", \"${admin_origin}\", \"*,roleSdk\")" box
+
+echo "Add cli api client"
+mysql -u root -p${mysql_root_password} \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-cli\", \"Cloudron Tool\", \"built-in\", \"secret-cli\", \"${admin_origin}\", \"*,roleSdk\")" box
 
 set_progress "80" "Starting Cloudron"
 systemctl start cloudron.target

setup/start/nginx/appconfig.ejs

@@ -24,6 +24,9 @@ server {
     ssl_ciphers 'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH';
     add_header Strict-Transport-Security "max-age=15768000; includeSubDomains";
 
+    # https://developer.mozilla.org/en-US/docs/Web/HTTP/X-Frame-Options
+    add_header X-Frame-Options SAMEORIGIN;
+
     proxy_http_version 1.1;
     proxy_intercept_errors on;
     proxy_read_timeout       3500;
@@ -44,6 +47,18 @@ server {
         return 307 <%= adminOrigin %>/appstatus.html?referrer=https://$host$request_uri;
     }
 
+<% if ( endpoint === 'app' ) { %>
+    # For some reason putting this webdav block inside location does not work
+    # http://serverfault.com/questions/121766/webdav-rename-fails-on-an-apache-mod-dav-install-behind-nginx
+    if ($request_method ~ ^(COPY|MOVE)$) {
+        set $destination $http_destination;
+    }
+    if ($destination ~* ^https(.+)$) {
+        set $destination http$1;
+    }
+    proxy_set_header Destination $destination;
+<% } %>
+
     location / {
         # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
         proxy_buffer_size       128k;
@@ -59,6 +74,7 @@ server {
             client_max_body_size 1m;
         }
 
+        # the read timeout is between successive reads and not the whole connection
         location ~ ^/api/v1/apps/.*/exec$ {
             proxy_pass   http://127.0.0.1:3000;
             proxy_read_timeout 30m;

setup/start/nginx/nginx.conf

@@ -24,7 +24,14 @@ http {
 
     sendfile        on;
 
-    keepalive_timeout  65;
+    # timeout for client to finish sending headers
+    client_header_timeout 30s;
+
+    # timeout for reading client request body (successive read timeout and not whole body!)
+    client_body_timeout 60s;
+
+    # keep-alive connections timeout in 65s. this is because many browsers timeout in 60 seconds
+    keepalive_timeout  65s;
 
     # HTTP server
     server {
@@ -50,22 +57,15 @@ http {
         }
     }
 
-    # We have to enable https for nginx to read in the vhost in http request
-    # and send a 404. This is a side-effect of using wildcard DNS
+    # This server handles the naked domain for custom domains.
+    # It can also be used for wildcard subdomain 404. This feature is not used by the Cloudron itself
+    # because box always sets up DNS records for app subdomains.
     server {
         listen 443 default_server;
         ssl on;
         ssl_certificate      cert/host.cert;
         ssl_certificate_key  cert/host.key;
 
-        # increase the proxy buffer sizes to not run into buffer issues (http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers)
-        proxy_buffer_size       128k;
-        proxy_buffers           4 256k;
-        proxy_busy_buffers_size 256k;
-
-        # Disable check to allow unlimited body sizes
-        client_max_body_size 0;
-
         error_page 404 = @fallback;
         location @fallback {
             internal;
@@ -79,6 +79,7 @@ http {
             rewrite ^/$ /nakeddomain.html break;
         }
 
+        # required for /api/v1/cloudron/avatar
         location /api/ {
             proxy_pass http://127.0.0.1:3000;
             client_max_body_size 1m;

src/INFRA_VERSION

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# If you change the infra version, be sure to put a warning
-# in the change log
-
-INFRA_VERSION=28
-
-# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
-# These constants are used in the installer script as well
-BASE_IMAGE=cloudron/base:0.8.0
-MYSQL_IMAGE=cloudron/mysql:0.11.0
-POSTGRESQL_IMAGE=cloudron/postgresql:0.9.0
-MONGODB_IMAGE=cloudron/mongodb:0.9.0
-REDIS_IMAGE=cloudron/redis:0.8.0 # if you change this, fix src/addons.js as well
-MAIL_IMAGE=cloudron/mail:0.11.0
-GRAPHITE_IMAGE=cloudron/graphite:0.8.0
-
-MYSQL_REPO=cloudron/mysql
-POSTGRESQL_REPO=cloudron/postgresql
-MONGODB_REPO=cloudron/mongodb
-REDIS_REPO=cloudron/redis # if you change this, fix src/addons.js as well
-MAIL_REPO=cloudron/mail
-GRAPHITE_REPO=cloudron/graphite

src/addons.js

@@ -1,15 +1,12 @@
 'use strict';
 
 exports = module.exports = {
-    initialize: initialize,
-
     setupAddons: setupAddons,
     teardownAddons: teardownAddons,
     backupAddons: backupAddons,
     restoreAddons: restoreAddons,
 
     getEnvironment: getEnvironment,
-    getLinksSync: getLinksSync,
     getBindsSync: getBindsSync,
     getContainerNamesSync: getContainerNamesSync,
 
@@ -21,29 +18,34 @@ exports = module.exports = {
 var appdb = require('./appdb.js'),
     assert = require('assert'),
     async = require('async'),
-    certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
-    DatabaseError = require('./databaseerror.js'),
+    ClientsError = clients.ClientsError,
     debug = require('debug')('box:addons'),
     docker = require('./docker.js'),
     dockerConnection = docker.connection,
     fs = require('fs'),
     generatePassword = require('password-generator'),
     hat = require('hat'),
+    infra = require('./infra_version.js'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
 
 var NOOP = function (app, options, callback) { return callback(); };
 
 // setup can be called multiple times for the same app (configure crash restart) and existing data must not be lost
 // teardown is destructive. app data stored with the addon is lost
 var KNOWN_ADDONS = {
+    email: {
+        setup: setupEmail,
+        teardown: teardownEmail,
+        backup: NOOP,
+        restore: setupEmail
+    },
     ldap: {
         setup: setupLdap,
         teardown: teardownLdap,
@@ -80,6 +82,12 @@ var KNOWN_ADDONS = {
         backup: backupPostgreSql,
         restore: restorePostgreSql
     },
+    recvmail: {
+        setup: setupRecvMail,
+        teardown: teardownRecvMail,
+        backup: NOOP,
+        restore: setupRecvMail
+    },
     redis: {
         setup: setupRedis,
         teardown: teardownRedis,
@@ -112,8 +120,7 @@ var KNOWN_ADDONS = {
     }
 };
 
-var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh'),
-    SETUP_INFRA_CMD = path.join(__dirname, 'scripts/setup_infra.sh');;
+var RMAPPDIR_CMD = path.join(__dirname, 'scripts/rmappdir.sh');
 
 function debugApp(app, args) {
     assert(!app || typeof app === 'object');
@@ -122,17 +129,6 @@ function debugApp(app, args) {
     debug(prefix + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
 }
 
-function initialize(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
-
-    debug('initializing addon infrastructure');
-    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
-        if (error) return callback(error);
-
-        shell.sudo('seutp_infra', [ SETUP_INFRA_CMD, config.fqdn(), config.adminFqdn(), certFilePath, keyFilePath ], callback);
-    });
-}
-
 function setupAddons(app, addons, callback) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
@@ -212,28 +208,6 @@ function getEnvironment(app, callback) {
     appdb.getAddonConfigByAppId(app.id, callback);
 }
 
-function getLinksSync(app, addons) {
-    assert.strictEqual(typeof app, 'object');
-    assert(!addons || typeof addons === 'object');
-
-    var links = [ ];
-
-    if (!addons) return links;
-
-    for (var addon in addons) {
-        switch (addon) {
-        case 'mysql': links.push('mysql:mysql'); break;
-        case 'postgresql': links.push('postgresql:postgresql'); break;
-        case 'sendmail': links.push('mail:mail'); break;
-        case 'redis': links.push('redis-' + app.id + ':redis-' + app.id); break;
-        case 'mongodb': links.push('mongodb:mongodb'); break;
-        default: break;
-        }
-    }
-
-    return links;
-}
-
 function getBindsSync(app, addons) {
     assert.strictEqual(typeof app, 'object');
     assert(!addons || typeof addons === 'object');
@@ -280,22 +254,18 @@ function setupOauth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    debugApp(app, 'setupOauth: id:%s clientSecret:%s', id, clientSecret);
+    clients.delByAppIdAndType(appId, clients.TYPE_OAUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(appId, clientdb.TYPE_OAUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_OAUTH, clientSecret, redirectURI, scope, function (error) {
+        clients.add(appId, clients.TYPE_OAUTH, redirectURI, scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'OAUTH_CLIENT_ID=' + id,
-                'OAUTH_CLIENT_SECRET=' + clientSecret,
+                'OAUTH_CLIENT_ID=' + result.id,
+                'OAUTH_CLIENT_SECRET=' + result.clientSecret,
                 'OAUTH_ORIGIN=' + config.adminOrigin()
             ];
 
@@ -313,8 +283,8 @@ function teardownOauth(app, options, callback) {
 
     debugApp(app, 'teardownOauth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_OAUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_OAUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'oauth', callback);
     });
@@ -326,23 +296,20 @@ function setupSimpleAuth(app, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var appId = app.id;
-    var id = 'cid-' + uuid.v4();
     var scope = 'profile';
 
-    debugApp(app, 'setupSimpleAuth: id:%s', id);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
+        if (error && error.reason !== ClientsError.NOT_FOUND) return callback(error);
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) { // remove existing creds
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return callback(error);
-
-        clientdb.add(id, appId, clientdb.TYPE_SIMPLE_AUTH, '', '', scope, function (error) {
+        clients.add(appId, clients.TYPE_SIMPLE_AUTH, '', scope, function (error, result) {
             if (error) return callback(error);
 
             var env = [
-                'SIMPLE_AUTH_SERVER=172.17.0.1',
+                'SIMPLE_AUTH_SERVER=172.18.0.1',
                 'SIMPLE_AUTH_PORT=' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_URL=http://172.17.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
-                'SIMPLE_AUTH_ORIGIN=http://172.17.0.1:' + config.get('simpleAuthPort'),
-                'SIMPLE_AUTH_CLIENT_ID=' + id
+                'SIMPLE_AUTH_URL=http://172.18.0.1:' + config.get('simpleAuthPort'), // obsolete, remove
+                'SIMPLE_AUTH_ORIGIN=http://172.18.0.1:' + config.get('simpleAuthPort'),
+                'SIMPLE_AUTH_CLIENT_ID=' + result.id
             ];
 
             debugApp(app, 'Setting simple auth addon config to %j', env);
@@ -359,26 +326,57 @@ function teardownSimpleAuth(app, options, callback) {
 
     debugApp(app, 'teardownSimpleAuth');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_SIMPLE_AUTH, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) console.error(error);
+    clients.delByAppIdAndType(app.id, clients.TYPE_SIMPLE_AUTH, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) console.error(error);
 
         appdb.unsetAddonConfig(app.id, 'simpleauth', callback);
     });
 }
 
+function setupEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    // note that "external" access info can be derived from MAIL_DOMAIN (since it's part of user documentation)
+    var env = [
+        'MAIL_SMTP_SERVER=mail',
+        'MAIL_SMTP_PORT=2525',
+        'MAIL_IMAP_SERVER=mail',
+        'MAIL_IMAP_PORT=9993',
+        'MAIL_SIEVE_SERVER=mail',
+        'MAIL_SIEVE_PORT=4190',
+        'MAIL_DOMAIN=' + config.fqdn()
+    ];
+
+    debugApp(app, 'Setting up Email');
+
+    appdb.setAddonConfig(app.id, 'email', env, callback);
+}
+
+function teardownEmail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Tearing down Email');
+
+    appdb.unsetAddonConfig(app.id, 'email', callback);
+}
+
 function setupLdap(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var env = [
-        'LDAP_SERVER=172.17.0.1',
+        'LDAP_SERVER=172.18.0.1',
         'LDAP_PORT=' + config.get('ldapPort'),
-        'LDAP_URL=ldap://172.17.0.1:' + config.get('ldapPort'),
+        'LDAP_URL=ldap://172.18.0.1:' + config.get('ldapPort'),
         'LDAP_USERS_BASE_DN=ou=users,dc=cloudron',
         'LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron',
         'LDAP_BIND_DN=cn='+ app.id + ',ou=apps,dc=cloudron',
-        'LDAP_BIND_PASSWORD=' + hat(256) // this is ignored
+        'LDAP_BIND_PASSWORD=' + hat(8 * 128) // this is ignored
     ];
 
     debugApp(app, 'Setting up LDAP');
@@ -401,20 +399,17 @@ function setupSendMail(app, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '-app';
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
 
-    var env = [
-        'MAIL_SMTP_SERVER=mail',
-        'MAIL_SMTP_PORT=2500', // if you change this, change the mail container
-        'MAIL_SMTP_USERNAME=' + from, // change this to app.id after apps have moved
-        'MAIL_SMTP_PASSWORD=' + 'app-' + app.id, // this is ignored
-        'MAIL_FROM=' + from + '@' + config.fqdn(),
-        'MAIL_DOMAIN=' + config.fqdn()
-    ];
+    var cmd = [ '/addons/mail/service.sh', 'add-send', from ];
 
-    debugApp(app, 'Setting up sendmail');
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
 
-    appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting sendmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'sendmail', env, callback);
+    });
 }
 
 function teardownSendMail(app, options, callback) {
@@ -424,7 +419,55 @@ function teardownSendMail(app, options, callback) {
 
     debugApp(app, 'Tearing down sendmail');
 
-    appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    var from = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-send', from ];
+
+    debugApp(app, 'Tearing down sendmail');
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'sendmail', callback);
+    });
+}
+
+function setupRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debugApp(app, 'Setting up recvmail');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'add-recv', to ];
+
+    docker.execContainer('mail', cmd, { bufferStdout: true }, function (error, stdout) {
+        if (error) return callback(error);
+
+        var env = stdout.toString('utf8').split('\n').slice(0, -1); // remove trailing newline
+        debugApp(app, 'Setting recvmail addon config to %j', env);
+        appdb.setAddonConfig(app.id, 'recvmail', env, callback);
+    });
+}
+
+function teardownRecvMail(app, options, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    var to = (app.location ? app.location : app.manifest.title.replace(/[^a-zA-Z0-9]/, '')) + '.app';
+
+    var cmd = [ '/addons/mail/service.sh', 'remove-recv', to ];
+
+    debugApp(app, 'Tearing down recvmail');
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(error);
+
+        appdb.unsetAddonConfig(app.id, 'recvmail', callback);
+    });
 }
 
 function setupMySql(app, options, callback) {
@@ -617,38 +660,6 @@ function restoreMongoDb(app, options, callback) {
     });
 }
 
-function forwardRedisPort(appId, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    dockerConnection.getContainer('redis-' + appId).inspect(function (error, data) {
-        if (error) return callback(new Error('Unable to inspect container:' + error));
-
-        var redisPort = parseInt(safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort'), 10);
-        if (!Number.isInteger(redisPort)) return callback(new Error('Unable to get container port mapping'));
-
-        return callback(null);
-    });
-}
-
-function stopAndRemoveRedis(container, callback) {
-    function ignoreError(func) {
-        return function (callback) {
-            func(function (error) {
-                if (error) debug('stopAndRemoveRedis: Ignored error:', error);
-                callback();
-            });
-        };
-    }
-
-    // stopping redis with SIGTERM makes it commit the database to disk
-    async.series([
-        ignoreError(container.stop.bind(container, { t: 10 })),
-        ignoreError(container.wait.bind(container)),
-        ignoreError(container.remove.bind(container, { force: true, v: true }))
-    ], callback);
-}
-
 // Ensures that app's addon redis container is running. Can be called when named container already exists/running
 function setupRedis(app, options, callback) {
     assert.strictEqual(typeof app, 'object');
@@ -665,34 +676,16 @@ function setupRedis(app, options, callback) {
 
     if (!safe.fs.mkdirSync(redisDataDir) && safe.error.code !== 'EEXIST') return callback(new Error('Error creating redis data dir:' + safe.error));
 
-    var createOptions = {
-        name: 'redis-' + app.id,
-        Hostname: 'redis-' + app.location,
-        Tty: true,
-        Image: 'cloudron/redis:0.8.0', // if you change this, fix src/INFRA_VERSION as well
-        Cmd: null,
-        Volumes: {
-            '/tmp': {},
-            '/run': {}
-        },
-        VolumesFrom: [],
-        HostConfig: {
-            Binds: [
-                redisVarsFile + ':/etc/redis/redis_vars.sh:ro',
-                redisDataDir + ':/var/lib/redis:rw'
-            ],
-            Memory: 1024 * 1024 * 75, // 100mb
-            MemorySwap: 1024 * 1024 * 75 * 2, // 150mb
-            PortBindings: {
-                '6379/tcp': [{ HostPort: '0', HostIp: '127.0.0.1' }]
-            },
-            ReadonlyRootfs: true,
-            RestartPolicy: {
-                'Name': 'always',
-                'MaximumRetryCount': 0
-            }
-        }
-    };
+    const tag = infra.images.redis.tag, redisName = 'redis-' + app.id;
+    const cmd = `docker run --restart=always -d --name=${redisName} \
+                --net cloudron \
+                --net-alias ${redisName} \
+                -m 100m \
+                --memory-swap 150m \
+                -p 6379:6379 \
+                -v ${redisVarsFile}:/etc/redis/redis_vars.sh:ro \
+                -v ${redisDataDir}:/var/lib/redis:rw \
+                --read-only -v /tmp -v /run ${tag}`;
 
     var env = [
         'REDIS_URL=redis://redisuser:' + redisPassword + '@redis-' + app.id,
@@ -701,21 +694,15 @@ function setupRedis(app, options, callback) {
         'REDIS_PORT=6379'
     ];
 
-    var redisContainer = dockerConnection.getContainer(createOptions.name);
-    stopAndRemoveRedis(redisContainer, function () {
-        dockerConnection.createContainer(createOptions, function (error) {
-            if (error && error.statusCode !== 409) return callback(error); // if not already created
-
-            redisContainer.start(function (error) {
-                if (error && error.statusCode !== 304) return callback(error); // if not already running
-
-                appdb.setAddonConfig(app.id, 'redis', env, function (error) {
-                    if (error) return callback(error);
-
-                    forwardRedisPort(app.id, callback);
-                });
-            });
-        });
+    async.series([
+        // stop so that redis can flush itself with SIGTERM
+        shell.execSync.bind(null, 'stopRedis', `docker stop --time=10 ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'stopRedis', `docker rm --volumes ${redisName} 2>/dev/null || true`),
+        shell.execSync.bind(null, 'startRedis', cmd),
+        appdb.setAddonConfig.bind(null, app.id, 'redis', env)
+    ], function (error) {
+        if (error) debug('Error setting up redis: ', error);
+        callback(error);
     });
 }
 

src/appdb.js

@@ -26,6 +26,7 @@ exports = module.exports = {
 
     // installation codes (keep in sync in UI)
     ISTATE_PENDING_INSTALL: 'pending_install', // installs and fresh reinstalls
+    ISTATE_PENDING_CLONE: 'pending_clone', // clone
     ISTATE_PENDING_CONFIGURE: 'pending_configure', // config (location, port) changes and on infra update
     ISTATE_PENDING_UNINSTALL: 'pending_uninstall', // uninstallation
     ISTATE_PENDING_RESTORE: 'pending_restore', // restore to previous backup or on upgrade
@@ -58,7 +59,7 @@ var assert = require('assert'),
 
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.installationProgress', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'apps.location', 'apps.dnsRecordId',
-    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.lastBackupConfigJson', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain' ].join(',');
+    'apps.accessRestrictionJson', 'apps.lastBackupId', 'apps.oldConfigJson', 'apps.memoryLimit', 'apps.altDomain' ].join(',');
 
 var PORT_BINDINGS_FIELDS = [ 'hostPort', 'environmentVariable', 'appId' ].join(',');
 
@@ -69,10 +70,6 @@ function postProcess(result) {
     result.manifest = safe.JSON.parse(result.manifestJson);
     delete result.manifestJson;
 
-    assert(result.lastBackupConfigJson === null || typeof result.lastBackupConfigJson === 'string');
-    result.lastBackupConfig = safe.JSON.parse(result.lastBackupConfigJson);
-    delete result.lastBackupConfigJson;
-
     assert(result.oldConfigJson === null || typeof result.oldConfigJson === 'string');
     result.oldConfig = safe.JSON.parse(result.oldConfigJson);
     delete result.oldConfigJson;
@@ -160,27 +157,31 @@ function getAll(callback) {
     });
 }
 
-function add(id, appStoreId, manifest, location, portBindings, accessRestriction, memoryLimit, altDomain, callback) {
+function add(id, appStoreId, manifest, location, portBindings, data, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof appStoreId, 'string');
     assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof manifest.version, 'string');
     assert.strictEqual(typeof location, 'string');
     assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert.strictEqual(typeof memoryLimit, 'number');
-    assert(altDomain === null || typeof altDomain === 'string');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     portBindings = portBindings || { };
 
     var manifestJson = JSON.stringify(manifest);
+
+    var accessRestriction = data.accessRestriction || null;
     var accessRestrictionJson = JSON.stringify(accessRestriction);
+    var memoryLimit = data.memoryLimit || 0;
+    var altDomain = data.altDomain || null;
+    var installationState = data.installationState || exports.ISTATE_PENDING_INSTALL;
+    var lastBackupId = data.lastBackupId || null; // used when cloning
 
     var queries = [ ];
     queries.push({
-        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
-        args: [ id, appStoreId, manifestJson, exports.ISTATE_PENDING_INSTALL, location, accessRestrictionJson, memoryLimit, altDomain ]
+        query: 'INSERT INTO apps (id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, lastBackupId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
+        args: [ id, appStoreId, manifestJson, installationState, location, accessRestrictionJson, memoryLimit, altDomain, lastBackupId ]
     });
 
     Object.keys(portBindings).forEach(function (env) {
@@ -284,9 +285,6 @@ function updateWithConstraints(id, app, constraints, callback) {
         if (p === 'manifest') {
             fields.push('manifestJson = ?');
             values.push(JSON.stringify(app[p]));
-        } else if (p === 'lastBackupConfig') {
-            fields.push('lastBackupConfigJson = ?');
-            values.push(JSON.stringify(app[p]));
         } else if (p === 'oldConfig') {
             fields.push('oldConfigJson = ?');
             values.push(JSON.stringify(app[p]));
@@ -345,14 +343,17 @@ function setInstallationCommand(appId, installationState, values, callback) {
     // uninstall is allowed in any state
     // force update is allowed in any state including pending_uninstall! (for better or worse)
     // restore is allowed from installed or error state
-    // update and configure are allowed only in installed state
+    // configure is allowed in installed state or currently configuring or in error state
+    // update and backup are allowed only in installed state
 
     if (installationState === exports.ISTATE_PENDING_UNINSTALL || installationState === exports.ISTATE_PENDING_FORCE_UPDATE) {
         updateWithConstraints(appId, values, '', callback);
     } else if (installationState === exports.ISTATE_PENDING_RESTORE) {
         updateWithConstraints(appId, values, 'AND (installationState = "installed" OR installationState = "error")', callback);
-    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_CONFIGURE || installationState === exports.ISTATE_PENDING_BACKUP) {
+    } else if (installationState === exports.ISTATE_PENDING_UPDATE || installationState === exports.ISTATE_PENDING_BACKUP || installationState === exports.ISTATE_PENDING_CONFIGURE) {
         updateWithConstraints(appId, values, 'AND installationState = "installed"', callback);
+    } else if (installationState === exports.ISTATE_PENDING_CONFIGURE) {
+        updateWithConstraints(appId, values, 'AND installationState = "installed" OR installationState = "pending_configure" OR installationState = "error"', callback);
     } else {
         callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, 'invalid installationState'));
     }

src/apphealthmonitor.js

@@ -17,7 +17,7 @@ exports = module.exports = {
 };
 
 var HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable
-var UNHEALTHY_THRESHOLD = 3 * 60 * 1000; // 3 minutes
+var UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes
 var gHealthInfo = { }; // { time, emailSent }
 var gRunTimeout = null;
 var gDockerEventStream = null;

src/apps.js

@@ -15,6 +15,7 @@ exports = module.exports = {
     uninstall: uninstall,
 
     restore: restore,
+    clone: clone,
 
     update: update,
 
@@ -30,7 +31,12 @@ exports = module.exports = {
 
     checkManifestConstraints: checkManifestConstraints,
 
-    autoupdateApps: autoupdateApps,
+    updateApps: updateApps,
+
+    restoreInstalledApps: restoreInstalledApps,
+    configureInstalledApps: configureInstalledApps,
+
+    getAppConfig: getAppConfig,
 
     // exported for testing
     _validateHostname: validateHostname,
@@ -43,6 +49,7 @@ var addons = require('./addons.js'),
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
+    BackupsError = backups.BackupsError,
     certificates = require('./certificates.js'),
     config = require('./config.js'),
     constants = require('./constants.js'),
@@ -62,6 +69,7 @@ var addons = require('./addons.js'),
     superagent = require('superagent'),
     taskmanager = require('./taskmanager.js'),
     util = require('util'),
+    uuid = require('node-uuid'),
     validator = require('validator');
 
 // http://dustinsenos.com/articles/customErrorsInNode
@@ -103,16 +111,16 @@ AppsError.BAD_CERTIFICATE = 'Invalid certificate';
 // https://en.wikipedia.org/wiki/Hostname#Restrictions_on_valid_host_names
 // We are validating the validity of the location-fqdn as host name
 function validateHostname(location, fqdn) {
-    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION ];
+    var RESERVED_LOCATIONS = [ constants.ADMIN_LOCATION, constants.API_LOCATION, constants.SMTP_LOCATION, constants.IMAP_LOCATION, constants.MAIL_LOCATION, constants.POSTMAN_LOCATION ];
 
-    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new Error(location + ' is reserved');
+    if (RESERVED_LOCATIONS.indexOf(location) !== -1) return new AppsError(AppsError.BAD_FIELD, location + ' is reserved');
 
     if (location === '') return null; // bare location
 
-    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new Error('Hostname length cannot be greater than 63');
-    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new Error('Hostname can only contain alphanumerics and hyphen');
-    if (location[0] === '-' || location[location.length-1] === '-') return new Error('Hostname cannot start or end with hyphen');
-    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new Error('FQDN length exceeds 253 characters');
+    if ((location.length + 1 /*+ hyphen */ + fqdn.indexOf('.')) > 63) return new AppsError(AppsError.BAD_FIELD, 'Hostname length cannot be greater than 63');
+    if (location.match(/^[A-Za-z0-9-]+$/) === null) return new AppsError(AppsError.BAD_FIELD, 'Hostname can only contain alphanumerics and hyphen');
+    if (location[0] === '-' || location[location.length-1] === '-') return new AppsError(AppsError.BAD_FIELD, 'Hostname cannot start or end with hyphen');
+    if (location.length + 1 /* hyphen */ + fqdn.length > 253) return new AppsError(AppsError.BAD_FIELD, 'FQDN length exceeds 253 characters');
 
     return null;
 }
@@ -137,10 +145,12 @@ function validatePortBindings(portBindings, tcpPorts) {
         2020, /* install server */
         config.get('port'), /* app server (lo) */
         config.get('sysadminPort'), /* sysadmin app server (lo) */
+        config.get('smtpPort'), /* internal smtp port (lo) */
         config.get('ldapPort'), /* ldap server (lo) */
         config.get('oauthProxyPort'), /* oauth proxy server (lo) */
         config.get('simpleAuthPort'), /* simple auth server (lo) */
         3306, /* mysql (lo) */
+        4190, /* managesieve */
         8000 /* graphite (lo) */
     ];
 
@@ -150,8 +160,8 @@ function validatePortBindings(portBindings, tcpPorts) {
     for (env in portBindings) {
         if (!/^[a-zA-Z0-9_]+$/.test(env)) return new AppsError(AppsError.BAD_FIELD, env + ' is not valid environment variable');
 
-        if (!Number.isInteger(portBindings[env])) return new Error(portBindings[env] + ' is not an integer');
-        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new Error(portBindings[env] + ' is out of range');
+        if (!Number.isInteger(portBindings[env])) return new AppsError(AppsError.BAD_FIELD, portBindings[env] + ' is not an integer');
+        if (portBindings[env] <= 0 || portBindings[env] > 65535) return new AppsError(AppsError.BAD_FIELD, portBindings[env] + ' is out of range');
 
         if (RESERVED_PORTS.indexOf(portBindings[env]) !== -1) return new AppsError(AppsError.PORT_RESERVED, String(portBindings[env]));
     }
@@ -174,19 +184,20 @@ function validateAccessRestriction(accessRestriction) {
     var noUsers = true, noGroups = true;
 
     if (accessRestriction.users) {
-        if (!Array.isArray(accessRestriction.users)) return new Error('users array property required');
-        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new Error('All users have to be strings');
+        if (!Array.isArray(accessRestriction.users)) return new AppsError(AppsError.BAD_FIELD, 'users array property required');
+        if (!accessRestriction.users.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All users have to be strings');
         noUsers = accessRestriction.users.length === 0;
     }
 
     if (accessRestriction.groups) {
-        if (!Array.isArray(accessRestriction.groups)) return new Error('groups array property required');
-        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new Error('All groups have to be strings');
+        if (!Array.isArray(accessRestriction.groups)) return new AppsError(AppsError.BAD_FIELD, 'groups array property required');
+        if (!accessRestriction.groups.every(function (e) { return typeof e === 'string'; })) return new AppsError(AppsError.BAD_FIELD, 'All groups have to be strings');
         noGroups = accessRestriction.groups.length === 0;
     }
 
-    if (noUsers && noGroups) return new Error('users and groups array cannot both be empty');
+    if (noUsers && noGroups) return new AppsError(AppsError.BAD_FIELD, 'users and groups array cannot both be empty');
 
+    // TODO: maybe validate if the users and groups actually exist
     return null;
 }
 
@@ -201,8 +212,8 @@ function validateMemoryLimit(manifest, memoryLimit) {
     // this is needed so an app update can change the value in the manifest, and if not set by the user, the new value should be used
     if (memoryLimit === 0) return null;
 
-    if (memoryLimit < min) return new Error('memoryLimit too small');
-    if (memoryLimit > max) return new Error('memoryLimit too large');
+    if (memoryLimit < min) return new AppsError(AppsError.BAD_FIELD, 'memoryLimit too small');
+    if (memoryLimit > max) return new AppsError(AppsError.BAD_FIELD, 'memoryLimit too large');
 
     return null;
 }
@@ -229,6 +240,17 @@ function getDuplicateErrorDetails(location, portBindings, error) {
     return new AppsError(AppsError.ALREADY_EXISTS);
 }
 
+function getAppConfig(app) {
+    return {
+        manifest: app.manifest,
+        location: app.location,
+        accessRestriction: app.accessRestriction,
+        portBindings: app.portBindings,
+        memoryLimit: app.memoryLimit,
+        altDomain: app.altDomain
+    };
+}
+
 function getIconUrlSync(app) {
     var iconPath = paths.APPICONS_DIR + '/' + app.id + '.png';
     return fs.existsSync(iconPath) ? '/api/v1/apps/' + app.id + '/icon' : null;
@@ -342,145 +364,182 @@ function purchase(appStoreId, callback) {
     });
 }
 
-function install(appId, appStoreId, manifest, location, portBindings, accessRestriction, icon, cert, key, memoryLimit, altDomain, auditSource, callback) {
-    assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof appStoreId, 'string');
-    assert(manifest && typeof manifest === 'object');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert(!icon || typeof icon === 'string');
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof memoryLimit, 'number');
-    assert(altDomain === null || typeof altDomain === 'string');
+function downloadManifest(appStoreId, manifest, callback) {
+    if (!appStoreId && !manifest) return callback(new AppsError(AppsError.BAD_FIELD, 'Neither manifest nor appStoreId provided'));
+
+    if (!appStoreId) return callback(null, '', manifest);
+
+    var parts = appStoreId.split('@');
+
+    var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
+
+    debug('downloading manifest from %s', url);
+
+    superagent.get(url).end(function (error, result) {
+        if (error && !error.response) return callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Network error downloading manifest:' + error.message));
+
+        if (result.statusCode !== 200) return callback(new AppsError(AppsError.BAD_FIELD, util.format('Failed to get app info from store.', result.statusCode, result.text)));
+
+        callback(null, parts[0], result.body.manifest);
+    });
+}
+
+function install(data, auditSource, callback) {
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var error = manifestFormat.parse(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
+    var location = data.location.toLowerCase(),
+        portBindings = data.portBindings || null,
+        accessRestriction = data.accessRestriction || null,
+        icon = data.icon || null,
+        cert = data.cert || null,
+        key = data.key || null,
+        memoryLimit = data.memoryLimit || 0,
+        altDomain = data.altDomain || null;
 
-    error = checkManifestConstraints(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+    assert(data.appStoreId || data.manifest); // atleast one of them is required
 
-    error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validatePortBindings(portBindings, manifest.tcpPorts);
-    if (error) return callback(error);
-
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validateMemoryLimit(manifest, memoryLimit);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    // memoryLimit might come in as 0 if not specified
-    memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-
-    if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
-
-    // singleUser mode requires accessRestriction to contain exactly one user
-    if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
-    if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
-
-    if (icon) {
-        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
-
-        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
-            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
-        }
-    }
-
-    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
-    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
-
-    debug('Will install app with id : ' + appId);
-
-    purchase(appStoreId, function (error) {
+    downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
         if (error) return callback(error);
 
-        appdb.add(appId, appStoreId, manifest, location.toLowerCase(), portBindings, accessRestriction, memoryLimit, altDomain, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
-            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+        error = manifestFormat.parse(manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error: ' + error.message));
 
-            // save cert to data/box/certs
-            if (cert && key) {
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        error = checkManifestConstraints(manifest);
+        if (error) return callback(error);
+
+        error = validateHostname(location, config.fqdn());
+        if (error) return callback(error);
+
+        error = validatePortBindings(portBindings, manifest.tcpPorts);
+        if (error) return callback(error);
+
+        error = validateAccessRestriction(accessRestriction);
+        if (error) return callback(error);
+
+        error = validateMemoryLimit(manifest, memoryLimit);
+        if (error) return callback(error);
+
+        // memoryLimit might come in as 0 if not specified
+        memoryLimit = memoryLimit || manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+
+        if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+
+        // singleUser mode requires accessRestriction to contain exactly one user
+        if (manifest.singleUser && accessRestriction === null) return callback(new AppsError(AppsError.USER_REQUIRED));
+        if (manifest.singleUser && accessRestriction.users.length !== 1) return callback(new AppsError(AppsError.USER_REQUIRED));
+
+        if (icon) {
+            if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+
+            if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
+                return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
             }
+        }
 
-            taskmanager.restartAppTask(appId);
+        error = certificates.validateCertificate(cert, key, config.appFqdn(location));
+        if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
 
-            eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+        var appId = uuid.v4();
+        debug('Will install app with id : ' + appId);
 
-            callback(null);
+        purchase(appStoreId, function (error) {
+            if (error) return callback(error);
+
+            var data = {
+                accessRestriction: accessRestriction,
+                memoryLimit: memoryLimit,
+                altDomain: altDomain
+            };
+
+            appdb.add(appId, appStoreId, manifest, location, portBindings, data, function (error) {
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                // save cert to data/box/certs
+                if (cert && key) {
+                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+                }
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_INSTALL, auditSource, { appId: appId, location: location, manifest: manifest });
+
+                callback(null, { id : appId });
+            });
         });
     });
 }
 
-function configure(appId, location, portBindings, accessRestriction, cert, key, memoryLimit, altDomain, auditSource, callback) {
+function configure(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof location, 'string');
-    assert.strictEqual(typeof portBindings, 'object');
-    assert.strictEqual(typeof accessRestriction, 'object');
-    assert(cert === null || typeof cert === 'string');
-    assert(key === null || typeof key === 'string');
-    assert.strictEqual(typeof memoryLimit, 'number');
-    assert(altDomain === null || typeof altDomain === 'string');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var error = validateHostname(location, config.fqdn());
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = validateAccessRestriction(accessRestriction);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-    error = certificates.validateCertificate(cert, key, config.appFqdn(location));
-    if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
-
-    if (altDomain !== null && !validator.isFQDN(altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
-
     appdb.get(appId, function (error, app) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        error = validatePortBindings(portBindings, app.manifest.tcpPorts);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-        error = validateMemoryLimit(app.manifest, memoryLimit);
-        if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
-
-        // memoryLimit might come in as 0 if not specified
-        memoryLimit = memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
-
-        // save cert to data/box/certs
-        if (cert && key) {
-            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
-            if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+        var location, portBindings, values = { };
+        if ('location' in data) {
+            location = values.location = data.location.toLowerCase();
+            error = validateHostname(values.location, config.fqdn());
+            if (error) return callback(error);
+        } else {
+            location = app.location;
         }
 
-        var values = {
-            location: location.toLowerCase(),
-            accessRestriction: accessRestriction,
-            portBindings: portBindings,
-            memoryLimit: memoryLimit,
-            altDomain: altDomain,
+        if ('accessRestriction' in data) {
+            values.accessRestriction = data.accessRestriction;
+            error = validateAccessRestriction(values.accessRestriction);
+            if (error) return callback(error);
+        }
 
-            oldConfig: {
-                location: app.location,
-                accessRestriction: app.accessRestriction,
-                portBindings: app.portBindings,
-                memoryLimit: app.memoryLimit,
-                altDomain: altDomain
+        if ('altDomain' in data) {
+            values.altDomain = data.altDomain;
+            if (values.altDomain !== null && !validator.isFQDN(values.altDomain)) return callback(new AppsError(AppsError.BAD_FIELD, 'Invalid alt domain'));
+        }
+
+        if ('portBindings' in data) {
+            portBindings = values.portBindings = data.portBindings;
+            error = validatePortBindings(values.portBindings, app.manifest.tcpPorts);
+            if (error) return callback(error);
+        } else {
+            portBindings = app.portBindings;
+        }
+
+        if ('memoryLimit' in data) {
+            values.memoryLimit = data.memoryLimit;
+            error = validateMemoryLimit(app.manifest, values.memoryLimit);
+            if (error) return callback(error);
+
+            // memoryLimit might come in as 0 if not specified
+            values.memoryLimit = values.memoryLimit || app.memoryLimit || app.manifest.memoryLimit || constants.DEFAULT_MEMORY_LIMIT;
+        }
+
+        // save cert to data/box/certs. TODO: move this to apptask when we have a real task queue
+        if ('cert' in data && 'key' in data) {
+            if (data.cert && data.key) {
+                error = certificates.validateCertificate(data.cert, data.key, config.appFqdn(location));
+                if (error) return callback(new AppsError(AppsError.BAD_CERTIFICATE, error.message));
+
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'), data.cert)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving cert: ' + safe.error.message));
+                if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'), data.key)) return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving key: ' + safe.error.message));
+            } else { // remove existing cert/key
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.cert'))) debug('Error removing cert: ' + safe.error.message);
+                if (!safe.fs.unlinkSync(path.join(paths.APP_CERTS_DIR, config.appFqdn(location) + '.key'))) debug('Error removing key: ' + safe.error.message);
             }
-        };
+        }
+
+        values.oldConfig = getAppConfig(app);
 
         debug('Will configure app with id:%s values:%j', appId, values);
 
         appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_CONFIGURE, values, function (error) {
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location.toLowerCase(), portBindings, error));
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
             if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
@@ -493,76 +552,75 @@ function configure(appId, location, portBindings, accessRestriction, cert, key,
     });
 }
 
-function update(appId, force, manifest, portBindings, icon, auditSource, callback) {
+function update(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
-    assert.strictEqual(typeof force, 'boolean');
-    assert(manifest && typeof manifest === 'object');
-    assert(typeof portBindings === 'object'); // can be null
-    assert(!icon || typeof icon === 'string');
+    assert(data && typeof data === 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     debug('Will update app with id:%s', appId);
 
-    var error = manifestFormat.parse(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
+    downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
+        if (error) return callback(error);
 
-    error = checkManifestConstraints(manifest);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed:' + error.message));
+        var values = { };
 
-    error = validatePortBindings(portBindings, manifest.tcpPorts);
-    if (error) return callback(new AppsError(AppsError.BAD_FIELD, error.message));
+        error = manifestFormat.parse(manifest);
+        if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest error:' + error.message));
 
-    if (icon) {
-        if (!validator.isBase64(icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
+        error = checkManifestConstraints(manifest);
+        if (error) return callback(error);
 
-        if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(icon, 'base64'))) {
-            return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
-        }
-    }
+        values.manifest = manifest;
 
-    appdb.get(appId, function (error, app) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
-        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
-
-        var appStoreId = app.appStoreId;
-
-        // prevent user from installing a app with different manifest id over an existing app
-        // this allows cloudron install -f --app <appid> for an app installed from the appStore
-        if (app.manifest.id !== manifest.id) {
-            if (!force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
-            // clear appStoreId so that this app does not get updates anymore. this will mark is a dev app
-            appStoreId = '';
+        if ('portBindings' in data) {
+            values.portBindings = data.portBindings;
+            error = validatePortBindings(data.portBindings, values.manifest.tcpPorts);
+            if (error) return callback(error);
         }
 
-        // Ensure we update the memory limit in case the new app requires more memory as a minimum
-        var memoryLimit = manifest.memoryLimit ? (app.memoryLimit < manifest.memoryLimit ? manifest.memoryLimit : app.memoryLimit) : app.memoryLimit;
+        if ('icon' in data) {
+            if (data.icon) {
+                if (!validator.isBase64(data.icon)) return callback(new AppsError(AppsError.BAD_FIELD, 'icon is not base64'));
 
-        var values = {
-            appStoreId: appStoreId,
-            manifest: manifest,
-            portBindings: portBindings,
-            memoryLimit: memoryLimit,
-
-            oldConfig: {
-                manifest: app.manifest,
-                portBindings: app.portBindings,
-                accessRestriction: app.accessRestriction,
-                memoryLimit: app.memoryLimit,
-                altDomain: app.altDomain
+                if (!safe.fs.writeFileSync(path.join(paths.APPICONS_DIR, appId + '.png'), new Buffer(data.icon, 'base64'))) {
+                    return callback(new AppsError(AppsError.INTERNAL_ERROR, 'Error saving icon:' + safe.error.message));
+                }
+            } else {
+                safe.fs.unlinkSync(path.join(paths.APPICONS_DIR, appId + '.png'));
             }
-        };
+        }
 
-        appdb.setInstallationCommand(appId, force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
-            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, portBindings, error));
+        appdb.get(appId, function (error, app) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND, 'No such app'));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            taskmanager.restartAppTask(appId);
+            // prevent user from installing a app with different manifest id over an existing app
+            // this allows cloudron install -f --app <appid> for an app installed from the appStore
+            if (app.manifest.id !== values.manifest.id) {
+                if (!data.force) return callback(new AppsError(AppsError.BAD_FIELD, 'manifest id does not match. force to override'));
+                // clear appStoreId so that this app does not get updates anymore. this will mark it as a dev app
+                values.appStoreId = '';
+            }
 
-            eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest });
+            // Ensure we update the memory limit in case the new app requires more memory as a minimum
+            if (values.manifest.memoryLimit && app.memoryLimit < values.manifest.memoryLimit) {
+                values.memoryLimit = values.manifest.memoryLimit;
+            }
 
-            callback(null);
+            values.oldConfig = getAppConfig(app);
+
+            appdb.setInstallationCommand(appId, data.force ? appdb.ISTATE_PENDING_FORCE_UPDATE : appdb.ISTATE_PENDING_UPDATE, values, function (error) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails('' /* location cannot conflict */, values.portBindings, error));
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_UPDATE, auditSource, { appId: appId, toManifest: manifest, fromManifest: app.manifest, force: data.force });
+
+                callback(null);
+            });
         });
     });
 }
@@ -612,8 +670,9 @@ function getLogs(appId, lines, follow, callback) {
     });
 }
 
-function restore(appId, auditSource, callback) {
+function restore(appId, data, auditSource, callback) {
     assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -623,43 +682,99 @@ function restore(appId, auditSource, callback) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
         if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-        // restore without a backup is the same as re-install
-        var restoreConfig = app.lastBackupConfig, values = { };
-        if (restoreConfig) {
-            // re-validate because this new box version may not accept old configs.
-            // if we restore location, it should be validated here as well
-            error = checkManifestConstraints(restoreConfig.manifest);
-            if (error) return callback(new AppsError(AppsError.BAD_FIELD, 'Manifest cannot be installed: ' + error.message));
+        // for empty or null backupId, use existing manifest to mimic a reinstall
+        var func = data.backupId ? backups.getRestoreConfig.bind(null, data.backupId) : function (next) { return next(null, { manifest: app.manifest }); };
 
-            error = validatePortBindings(restoreConfig.portBindings, restoreConfig.manifest.tcpPorts); // maybe new ports got reserved now
-            if (error) return callback(error);
-
-            // ## should probably query new location, access restriction from user
-            values = {
-                manifest: restoreConfig.manifest,
-                portBindings: restoreConfig.portBindings,
-                memoryLimit: restoreConfig.memoryLimit,
-
-                oldConfig: {
-                    location: app.location,
-                    accessRestriction: app.accessRestriction,
-                    portBindings: app.portBindings,
-                    memoryLimit: app.memoryLimit,
-                    manifest: app.manifest,
-                    altDomain: app.altDomain
-                }
-            };
-        }
-
-        appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+        func(function (error, restoreConfig) {
+            if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
             if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
-            taskmanager.restartAppTask(appId);
+            if (!restoreConfig) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config'));
 
-            eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { appId: appId });
+            // re-validate because this new box version may not accept old configs
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(error);
 
-            callback(null);
+            var values = {
+                lastBackupId: data.backupId || null, // when null, apptask simply reinstalls
+                manifest: restoreConfig.manifest,
+
+                oldConfig: getAppConfig(app)
+            };
+
+            appdb.setInstallationCommand(appId, appdb.ISTATE_PENDING_RESTORE, values, function (error) {
+                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.BAD_STATE)); // might be a bad guess
+                if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                taskmanager.restartAppTask(appId);
+
+                eventlog.add(eventlog.ACTION_APP_RESTORE, auditSource, { appId: appId });
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function clone(appId, data, auditSource, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will clone app with id:%s', appId);
+
+    var location = data.location.toLowerCase(),
+        portBindings = data.portBindings || null,
+        backupId = data.backupId;
+
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof portBindings, 'object');
+
+    appdb.get(appId, function (error, app) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new AppsError(AppsError.NOT_FOUND));
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        backups.getRestoreConfig(backupId, function (error, restoreConfig) {
+            if (error && error.reason === BackupsError.EXTERNAL_ERROR) return callback(new AppsError(AppsError.EXTERNAL_ERROR, error.message));
+            if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+            if (!restoreConfig) callback(new AppsError(AppsError.EXTERNAL_ERROR, 'Could not get restore config'));
+
+            // re-validate because this new box version may not accept old configs
+            error = checkManifestConstraints(restoreConfig.manifest);
+            if (error) return callback(error);
+
+            error = validateHostname(location, config.fqdn());
+            if (error) return callback(error);
+
+            error = validatePortBindings(portBindings, restoreConfig.manifest.tcpPorts);
+            if (error) return callback(error);
+
+            var newAppId = uuid.v4(), appStoreId = app.appStoreId, manifest = restoreConfig.manifest;
+
+            purchase(appStoreId, function (error) {
+                if (error) return callback(error);
+
+                var data = {
+                    installationState: appdb.ISTATE_PENDING_CLONE,
+                    memoryLimit: app.memoryLimit,
+                    accessRestriction: app.accessRestriction,
+                    lastBackupId: backupId
+                };
+
+                appdb.add(newAppId, appStoreId, manifest, location, portBindings, data, function (error) {
+                    if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(getDuplicateErrorDetails(location, portBindings, error));
+                    if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+                    taskmanager.restartAppTask(newAppId);
+
+                    eventlog.add(eventlog.ACTION_APP_CLONE, auditSource, { appId: newAppId, oldAppId: appId, backupId: backupId, location: location, manifest: manifest });
+
+                    callback(null, { id : newAppId });
+                });
+            });
         });
     });
 }
@@ -716,14 +831,16 @@ function stop(appId, callback) {
 }
 
 function checkManifestConstraints(manifest) {
-    if (!manifest.dockerImage) return new Error('Missing dockerImage'); // dockerImage is optional in manifest
+    assert(manifest && typeof manifest === 'object');
+
+    if (!manifest.dockerImage) return new AppsError(AppsError.BAD_FIELD, 'Missing dockerImage'); // dockerImage is optional in manifest
 
     if (semver.valid(manifest.maxBoxVersion) && semver.gt(config.version(), manifest.maxBoxVersion)) {
-        return new Error('Box version exceeds Apps maxBoxVersion');
+        return new AppsError(AppsError.BAD_FIELD, 'Box version exceeds Apps maxBoxVersion');
     }
 
     if (semver.valid(manifest.minBoxVersion) && semver.gt(manifest.minBoxVersion, config.version())) {
-        return new Error('minBoxVersion exceeds Box version');
+        return new AppsError(AppsError.BAD_FIELD, 'minBoxVersion exceeds Box version');
     }
 
     return null;
@@ -747,10 +864,14 @@ function exec(appId, options, callback) {
 
         var container = docker.connection.getContainer(app.containerId);
 
-       var execOptions = {
+        var execOptions = {
             AttachStdin: true,
             AttachStdout: true,
             AttachStderr: true,
+            // A pseudo tty is a terminal which processes can detect (for example, disable colored output)
+            // Creating a pseudo terminal also assigns a terminal driver which detects control sequences
+            // When passing binary data, tty must be disabled. In addition, the stdout/stderr becomes a single
+            // unified stream because of the nature of a tty (see https://github.com/docker/docker/issues/19696)
             Tty: options.tty,
             Cmd: cmd
         };
@@ -760,9 +881,18 @@ function exec(appId, options, callback) {
             var startOptions = {
                 Detach: false,
                 Tty: options.tty,
-                stdin: true // this is a dockerode option that enabled openStdin in the modem
+                // hijacking upgrades the docker connection from http to tcp. because of this upgrade,
+                // we can work with half-close connections (not defined in http). this way, the client
+                // can properly signal that stdin is EOF by closing it's side of the socket. In http,
+                // the whole connection will be dropped when stdin get EOF.
+                // https://github.com/apocas/dockerode/commit/b4ae8a03707fad5de893f302e4972c1e758592fe
+                hijack: true,
+                stream: true,
+                stdin: true,
+                stdout: true,
+                stderr: true
             };
-            exec.start(startOptions, function(error, stream) {
+            exec.start(startOptions, function(error, stream /* in hijacked mode, this is a net.socket */) {
                 if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
 
                 if (options.rows && options.columns) {
@@ -775,23 +905,25 @@ function exec(appId, options, callback) {
     });
 }
 
-function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { manifest } }
+function updateApps(updateInfo, auditSource, callback) { // updateInfo is { appId -> { manifest } }
     assert.strictEqual(typeof updateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     function canAutoupdateApp(app, newManifest) {
-        var tcpPorts = newManifest.tcpPorts || { };
+        var newTcpPorts = newManifest.tcpPorts || { };
+        var oldTcpPorts = app.manifest.tcpPorts || { };
         var portBindings = app.portBindings; // this is never null
 
-        if (Object.keys(tcpPorts).length === 0 && Object.keys(portBindings).length === 0) return null;
-        if (Object.keys(tcpPorts).length === 0) return new Error('tcpPorts is now empty but portBindings is not');
-        if (Object.keys(portBindings).length === 0) return new Error('portBindings is now empty but tcpPorts is not');
-
-        for (var env in tcpPorts) {
-            if (!(env in portBindings)) return new Error(env + ' is required from user');
+        for (var env in newTcpPorts) {
+            if (!(env in oldTcpPorts)) return new Error(env + ' is required from user');
         }
 
-        // it's fine if one or more keys got removed
+        for (env in portBindings) {
+            if (!(env in newTcpPorts)) return new Error(env + ' was in use but new update removes it');
+        }
+
+        // it's fine if one or more (unused) keys got removed
         return null;
     }
 
@@ -810,8 +942,12 @@ function autoupdateApps(updateInfo, callback) { // updateInfo is { appId -> { ma
                 return iteratorDone();
             }
 
-           update(appId, false /* force */, updateInfo[appId].manifest, app.portBindings,
-                  null /* icon */, { userId: null, username: 'autoupdater' }, function (error) {
+            var data = {
+                manifest: updateInfo[appId].manifest,
+                force: false
+            };
+
+            update(appId, data, auditSource, function (error) {
                 if (error) debug('Error initiating autoupdate of %s. %s', appId, error.message);
 
                 iteratorDone(null);
@@ -858,3 +994,39 @@ function listBackups(page, perPage, appId, callback) {
         });
     });
 }
+
+function restoreInstalledApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        async.map(apps, function (app, iteratorDone) {
+            debug('marking %s for restore', app.location || app.id);
+
+            appdb.setInstallationCommand(app.id, appdb.ISTATE_PENDING_RESTORE, { oldConfig: null }, function (error) {
+                if (error) debug('did not mark %s for restore', app.location || app.id, error);
+
+                iteratorDone(); // always succeed
+            });
+        }, callback);
+    });
+}
+
+function configureInstalledApps(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    appdb.getAll(function (error, apps) {
+        if (error) return callback(new AppsError(AppsError.INTERNAL_ERROR, error));
+
+        async.map(apps, function (app, iteratorDone) {
+            debug('marking %s for reconfigure', app.location || app.id);
+
+            appdb.setInstallationCommand(app.id, appdb.ISTATE_PENDING_CONFIGURE, { oldConfig: null }, function (error) {
+                if (error) debug('did not mark %s for reconfigure', app.location || app.id, error);
+
+                iteratorDone(); // always succeed
+            });
+        }, callback);
+    });
+}

src/apptask.js

@@ -36,15 +36,14 @@ var addons = require('./addons.js'),
     async = require('async'),
     backups = require('./backups.js'),
     certificates = require('./certificates.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    ClientsError = clients.ClientsError,
     database = require('./database.js'),
-    DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:apptask'),
     docker = require('./docker.js'),
     ejs = require('ejs'),
     fs = require('fs'),
-    hat = require('hat'),
     manifestFormat = require('cloudron-manifestformat'),
     net = require('net'),
     nginx = require('./nginx.js'),
@@ -57,7 +56,6 @@ var addons = require('./addons.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
     util = require('util'),
-    uuid = require('node-uuid'),
     waitForDns = require('./waitfordns.js'),
     _ = require('underscore');
 
@@ -163,20 +161,18 @@ function allocateOAuthProxyCredentials(app, callback) {
 
     if (!nginx.requiresOAuthProxy(app)) return callback(null);
 
-    var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
     var redirectURI = 'https://' + config.appFqdn(app.location);
     var scope = 'profile';
 
-    clientdb.add(id, app.id, clientdb.TYPE_PROXY, clientSecret, redirectURI, scope, callback);
+    clients.add(app.id, clients.TYPE_PROXY, redirectURI, scope, callback);
 }
 
 function removeOAuthProxyCredentials(app, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.delByAppIdAndType(app.id, clientdb.TYPE_PROXY, function (error) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) {
+    clients.delByAppIdAndType(app.id, clients.TYPE_PROXY, function (error) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) {
             debugApp(app, 'Error removing OAuth client id', error);
             return callback(error);
         }
@@ -417,7 +413,7 @@ function install(app, callback) {
         updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
         exports._waitForAltDomainDnsPropagation.bind(null, app), // required when restoring and !lastBackupId
 
         updateApp.bind(null, app, { installationProgress: '95, Configure nginx' }),
@@ -443,7 +439,7 @@ function backup(app, callback) {
 
     async.series([
         updateApp.bind(null, app, { installationProgress: '10, Backing up' }),
-        backups.backupApp.bind(null, app, app.manifest.addons),
+        backups.backupApp.bind(null, app, app.manifest),
 
         // done!
         function (callback) {
@@ -521,7 +517,7 @@ function restore(app, callback) {
         updateApp.bind(null, app, { installationProgress: '85, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '90, Waiting for Alt Domain DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '90, Waiting for External Domain CNAME setup' }),
         exports._waitForAltDomainDnsPropagation.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '95, Configuring Nginx' }),
@@ -583,7 +579,7 @@ function configure(app, callback) {
         updateApp.bind(null, app, { installationProgress: '80, Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
-        updateApp.bind(null, app, { installationProgress: '85, Waiting for Alt Domain DNS propagation' }),
+        updateApp.bind(null, app, { installationProgress: '85, Waiting for External Domain CNAME setup' }),
         exports._waitForAltDomainDnsPropagation.bind(null, app),
 
         updateApp.bind(null, app, { installationProgress: '90, Configuring Nginx' }),
@@ -629,7 +625,6 @@ function update(app, callback) {
         removeCollectdProfile.bind(null, app),
         stopApp.bind(null, app),
         deleteContainers.bind(null, app),
-        addons.teardownAddons.bind(null, app, unusedAddons),
         function deleteImageIfChanged(done) {
              if (app.oldConfig.manifest.dockerImage === app.manifest.dockerImage) return done();
 
@@ -642,10 +637,13 @@ function update(app, callback) {
 
             async.series([
                 updateApp.bind(null, app, { installationProgress: '30, Backup app' }),
-                backups.backupApp.bind(null, app, app.oldConfig.manifest.addons)
+                backups.backupApp.bind(null, app, app.oldConfig.manifest)
             ], next);
         },
 
+        // only delete unused addons after backup
+        addons.teardownAddons.bind(null, app, unusedAddons),
+
         updateApp.bind(null, app, { installationProgress: '45, Downloading icon' }),
         downloadIcon.bind(null, app),
 
@@ -780,6 +778,7 @@ function startTask(appId, callback) {
         case appdb.ISTATE_PENDING_BACKUP: return backup(app, callback);
         case appdb.ISTATE_INSTALLED: return handleRunCommand(app, callback);
         case appdb.ISTATE_PENDING_INSTALL: return install(app, callback);
+        case appdb.ISTATE_PENDING_CLONE: return restore(app, callback);
         case appdb.ISTATE_PENDING_FORCE_UPDATE: return update(app, callback);
         case appdb.ISTATE_ERROR:
             debugApp(app, 'Internal error. apptask launched with error status.');
@@ -796,6 +795,11 @@ if (require.main === module) {
 
     debug('Apptask for %s', process.argv[2]);
 
+    process.on('SIGTERM', function () {
+        debug('taskmanager sent SIGTERM since it got a new task for this app');
+        process.exit(0);
+    });
+
     initialize(function (error) {
         if (error) throw error;
 

src/auth.js

@@ -10,17 +10,16 @@ exports = module.exports = {
 var assert = require('assert'),
     BasicStrategy = require('passport-http').BasicStrategy,
     BearerStrategy = require('passport-http-bearer').Strategy,
-    clientdb = require('./clientdb'),
+    clients = require('./clients'),
     ClientPasswordStrategy = require('passport-oauth2-client-password').Strategy,
+    ClientsError = clients.ClientsError,
     DatabaseError = require('./databaseerror'),
     debug = require('debug')('box:auth'),
     LocalStrategy = require('passport-local').Strategy,
     crypto = require('crypto'),
-    groups = require('./groups'),
     passport = require('passport'),
     tokendb = require('./tokendb'),
     user = require('./user'),
-    userdb = require('./userdb'),
     UserError = user.UserError,
     _ = require('underscore');
 
@@ -32,7 +31,7 @@ function initialize(callback) {
     });
 
     passport.deserializeUser(function(userId, callback) {
-        userdb.get(userId, function (error, result) {
+        user.get(userId, function (error, result) {
             if (error) return callback(error);
 
             var md5 = crypto.createHash('md5').update(result.email.toLowerCase()).digest('hex');
@@ -67,8 +66,8 @@ function initialize(callback) {
             debug('BasicStrategy: detected client id %s instead of username:password', username);
             // username is actually client id here
             // password is client secret
-            clientdb.get(username, function (error, client) {
-                if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+            clients.get(username, function (error, client) {
+                if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
                 if (client.clientSecret != password) return callback(null, false);
                 return callback(null, client);
@@ -85,8 +84,8 @@ function initialize(callback) {
     }));
 
     passport.use(new ClientPasswordStrategy(function (clientId, clientSecret, callback) {
-        clientdb.get(clientId, function(error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function(error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) { return callback(error); }
             if (client.clientSecret != clientSecret) { return callback(null, false); }
             return callback(null, client);
@@ -101,37 +100,12 @@ function initialize(callback) {
             // scopes here can define what capabilities that token carries
             // passport put the 'info' object into req.authInfo, where we can further validate the scopes
             var info = { scope: token.scope };
-            var tokenType;
 
-            if (token.identifier.indexOf(tokendb.PREFIX_DEV) === 0) {
-                token.identifier = token.identifier.slice(tokendb.PREFIX_DEV.length);
-                tokenType = tokendb.TYPE_DEV;
-            } else if (token.identifier.indexOf(tokendb.PREFIX_APP) === 0) {
-                tokenType = tokendb.TYPE_APP;
-                return callback(null, { id: token.identifier.slice(tokendb.PREFIX_APP.length), tokenType: tokenType }, info);
-            } else if (token.identifier.indexOf(tokendb.PREFIX_USER) === 0) {
-                tokenType = tokendb.TYPE_USER;
-                token.identifier = token.identifier.slice(tokendb.PREFIX_USER.length);
-            } else {
-                // legacy tokens assuming a user access token
-                tokenType = tokendb.TYPE_USER;
-            }
-
-            userdb.get(token.identifier, function (error, user) {
+            user.get(token.identifier, function (error, user) {
                 if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
                 if (error) return callback(error);
 
-                // amend the tokenType of the token owner
-                user.tokenType = tokenType;
-
-                // amend the admin flag
-                groups.isMember(groups.ADMIN_GROUP_ID, user.id, function (error, isAdmin) {
-                    if (error) return callback(error);
-
-                    user.admin = isAdmin;
-
-                    callback(null, user, info);
-                });
+                callback(null, user, info);
             });
         });
     }));

src/backups.js

@@ -7,6 +7,7 @@ exports = module.exports = {
     getByAppIdPaged: getByAppIdPaged,
 
     getRestoreUrl: getRestoreUrl,
+    getRestoreConfig: getRestoreConfig,
 
     ensureBackup: ensureBackup,
 
@@ -36,6 +37,7 @@ var addons = require('./addons.js'),
     safe = require('safetydance'),
     shell = require('./shell.js'),
     settings = require('./settings.js'),
+    superagent = require('superagent'),
     util = require('util'),
     webhooks = require('./webhooks.js');
 
@@ -135,12 +137,13 @@ function getBoxBackupCredentials(appBackupIds, callback) {
     });
 }
 
-function getAppBackupCredentials(app, callback) {
+function getAppBackupCredentials(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var now = new Date();
-    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), app.manifest.version);
+    var filebase = util.format('appbackup_%s_%s-v%s', app.id, now.toISOString(), manifest.version);
     var configFilename = filebase + '.json', dataFilename = filebase + '.tar.gz';
 
     settings.getBackupConfig(function (error, backupConfig) {
@@ -161,6 +164,32 @@ function getAppBackupCredentials(app, callback) {
     });
 }
 
+// backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
+function getRestoreConfig(backupId, callback) {
+    assert.strictEqual(typeof backupId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var configFile = backupId.replace(/\.tar\.gz$/, '.json');
+
+    settings.getBackupConfig(function (error, backupConfig) {
+        if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
+
+        api(backupConfig.provider).getRestoreUrl(backupConfig, configFile, function (error, result) {
+            if (error) return callback(error);
+
+            superagent.get(result.url).buffer(true).end(function (error, response) {
+                if (error && !error.response) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, error.message));
+                if (response.statusCode !== 200) return callback(new Error('Invalid response code when getting config.json : ' + response.statusCode));
+
+                var config = safe.JSON.parse(response.text);
+                if (!config) return callback(new BackupsError(BackupsError.EXTERNAL_ERROR, 'Error in config:' + safe.error.message));
+
+                return callback(null, config);
+            });
+        });
+    });
+}
+
 // backupId is the s3 filename. appbackup_%s_%s-v%s.tar.gz
 function getRestoreUrl(backupId, callback) {
     assert.strictEqual(typeof backupId, 'string');
@@ -185,14 +214,15 @@ function getRestoreUrl(backupId, callback) {
     });
 }
 
-function copyLastBackup(app, callback) {
+function copyLastBackup(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifeset === 'object');
     assert.strictEqual(typeof callback, 'function');
 
     var now = new Date();
-    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), app.manifest.version);
-    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), app.manifest.version);
+    var toFilenameArchive = util.format('appbackup_%s_%s-v%s.tar.gz', app.id, now.toISOString(), manifest.version);
+    var toFilenameConfig = util.format('appbackup_%s_%s-v%s.json', app.id, now.toISOString(), manifest.version);
 
     settings.getBackupConfig(function (error, backupConfig) {
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
@@ -267,11 +297,12 @@ function canBackupApp(app) {
 
 // set the 'creation' date of lastBackup so that the backup persists across time based archival rules
 // s3 does not allow changing creation time, so copying the last backup is easy way out for now
-function reuseOldAppBackup(app, callback) {
+function reuseOldAppBackup(app, manifest, callback) {
     assert.strictEqual(typeof app.lastBackupId, 'string');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    copyLastBackup(app, function (error, newBackupId) {
+    copyLastBackup(app, manifest, function (error, newBackupId) {
         if (error) return callback(error);
 
         debugApp(app, 'reuseOldAppBackup: reused old backup %s as %s', app.lastBackupId, newBackupId);
@@ -280,12 +311,12 @@ function reuseOldAppBackup(app, callback) {
     });
 }
 
-function createNewAppBackup(app, addonsToBackup, callback) {
+function createNewAppBackup(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getAppBackupCredentials(app, function (error, result) {
+    getAppBackupCredentials(app, manifest, function (error, result) {
         if (error) return callback(error);
 
         debugApp(app, 'createNewAppBackup: backup url:%s backup config url:%s', result.s3DataUrl, result.s3ConfigUrl);
@@ -294,14 +325,14 @@ function createNewAppBackup(app, addonsToBackup, callback) {
                      result.sessionToken, result.region, result.backupKey ];
 
         async.series([
-            addons.backupAddons.bind(null, app, addonsToBackup),
+            addons.backupAddons.bind(null, app, manifest.addons),
             shell.sudo.bind(null, 'backupApp', [ BACKUP_APP_CMD ].concat(args))
         ], function (error) {
             if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
             debugApp(app, 'createNewAppBackup: %s done', result.id);
 
-            backupdb.add({ id: result.id, version: app.manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
+            backupdb.add({ id: result.id, version: manifest.version, type: backupdb.BACKUP_TYPE_APP, dependsOn: [ ] }, function (error) {
                 if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
                 callback(null, result.id);
@@ -310,13 +341,12 @@ function createNewAppBackup(app, addonsToBackup, callback) {
     });
 }
 
-function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
+function setRestorePoint(appId, lastBackupId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof lastBackupId, 'string');
-    assert.strictEqual(typeof lastBackupConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    appdb.update(appId, { lastBackupId: lastBackupId, lastBackupConfig: lastBackupConfig }, function (error) {
+    appdb.update(appId, { lastBackupId: lastBackupId }, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new BackupsError(BackupsError.NOT_FOUND, 'No such app'));
         if (error) return callback(new BackupsError(BackupsError.INTERNAL_ERROR, error));
 
@@ -324,12 +354,12 @@ function setRestorePoint(appId, lastBackupId, lastBackupConfig, callback) {
     });
 }
 
-function backupApp(app, addonsToBackup, callback) {
+function backupApp(app, manifest, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert(!addonsToBackup || typeof addonsToBackup, 'object');
+    assert(manifest && typeof manifest === 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    var appConfig = null, backupFunction;
+    var backupFunction;
 
     if (!canBackupApp(app)) {
         if (!app.lastBackupId) {
@@ -337,17 +367,11 @@ function backupApp(app, addonsToBackup, callback) {
             return callback(new BackupsError(BackupsError.BAD_STATE, 'App not healthy and never backed up previously'));
         }
 
-        appConfig = app.lastBackupConfig;
-        backupFunction = reuseOldAppBackup.bind(null, app);
+        backupFunction = reuseOldAppBackup.bind(null, app, manifest);
     } else {
-        appConfig = {
-            manifest: app.manifest,
-            location: app.location,
-            portBindings: app.portBindings,
-            accessRestriction: app.accessRestriction,
-            memoryLimit: app.memoryLimit
-        };
-        backupFunction = createNewAppBackup.bind(null, app, addonsToBackup);
+        var appConfig = apps.getAppConfig(app);
+        appConfig.manifest = manifest;
+        backupFunction = createNewAppBackup.bind(null, app, manifest);
 
         if (!safe.fs.writeFileSync(path.join(paths.DATA_DIR, app.id + '/config.json'), JSON.stringify(appConfig), 'utf8')) {
             return callback(safe.error);
@@ -359,7 +383,7 @@ function backupApp(app, addonsToBackup, callback) {
 
         debugApp(app, 'backupApp: successful id:%s', backupId);
 
-        setRestorePoint(app.id, backupId, appConfig, function (error) {
+        setRestorePoint(app.id, backupId, function (error) {
             if (error) return callback(error);
 
             return callback(null, backupId);
@@ -386,7 +410,7 @@ function backupBoxAndApps(auditSource, callback) {
         async.mapSeries(allApps, function iterator(app, iteratorCallback) {
             ++processed;
 
-            backupApp(app, app.manifest.addons, function (error, backupId) {
+            backupApp(app, app.manifest, function (error, backupId) {
                 if (error && error.reason !== BackupsError.BAD_STATE) {
                     debugApp(app, 'Unable to backup', error);
                     return iteratorCallback(error);
@@ -433,8 +457,8 @@ function backup(auditSource, callback) {
     callback(null);
 }
 
-function ensureBackup(callback) {
-    callback = callback || NOOP_CALLBACK;
+function ensureBackup(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
 
     getPaged(1, 1, function (error, backups) {
         if (error) {
@@ -447,8 +471,7 @@ function ensureBackup(callback) {
             return callback(null);
         }
 
-        var eventSource = { userId: null, username: 'cron' };
-        backup(eventSource, callback);
+        backup(auditSource, callback);
     });
 }
 

src/certificates.js

@@ -2,7 +2,7 @@
 
 exports = module.exports = {
     installAdminCertificate: installAdminCertificate,
-    autoRenew: autoRenew,
+    renewAll: renewAll,
     setFallbackCertificate: setFallbackCertificate,
     setAdminCertificate: setAdminCertificate,
     CertificatesError: CertificatesError,
@@ -69,7 +69,8 @@ function getApi(app, callback) {
         var api = !app.altDomain && tlsConfig.provider === 'caas' ? caas : acme;
 
         var options = { };
-        options.prod = tlsConfig.provider.match(/.*-prod/) !== null;
+        // used by acme backend to determine the LE origin.
+        options.prod = (api === caas) ? !config.isDev() : tlsConfig.provider.match(/.*-prod/) !== null;
 
         // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
         // we cannot use admin@fqdn because the user might not have set it up.
@@ -123,9 +124,11 @@ function isExpiringSync(certFilePath, hours) {
     return result.status === 1; // 1 - expired 0 - not expired
 }
 
-function autoRenew(callback) {
-    debug('autoRenew: Checking certificates for renewal');
-    callback = callback || NOOP_CALLBACK;
+function renewAll(auditSource, callback) {
+    assert.strictEqual(typeof auditSource, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('renewAll: Checking certificates for renewal');
 
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
@@ -139,7 +142,7 @@ function autoRenew(callback) {
             var keyFilePath = path.join(paths.APP_CERTS_DIR, appDomain + '.key');
 
             if (!safe.fs.existsSync(keyFilePath)) {
-                debug('autoRenew: no existing key file for %s. skipping', appDomain);
+                debug('renewAll: no existing key file for %s. skipping', appDomain);
                 continue;
             }
 
@@ -148,7 +151,7 @@ function autoRenew(callback) {
             }
         }
 
-        debug('autoRenew: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
+        debug('renewAll: %j needs to be renewed', expiringApps.map(function (a) { return a.altDomain || config.appFqdn(a.location); }));
 
         async.eachSeries(expiringApps, function iterator(app, iteratorCallback) {
             var domain = app.altDomain || config.appFqdn(app.location);
@@ -156,28 +159,28 @@ function autoRenew(callback) {
             getApi(app, function (error, api, apiOptions) {
                 if (error) return callback(error);
 
-                debug('autoRenew: renewing cert for %s with options %j', domain, apiOptions);
+                debug('renewAll: renewing cert for %s with options %j', domain, apiOptions);
 
                 api.getCertificate(domain, apiOptions, function (error) {
                     var certFilePath = path.join(paths.APP_CERTS_DIR, domain + '.cert');
                     var keyFilePath = path.join(paths.APP_CERTS_DIR, domain + '.key');
 
                     var errorMessage = error ? error.message : '';
-                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, { userId: null, username: 'cron' }, { domain: domain, errorMessage: errorMessage });
+                    eventlog.add(eventlog.ACTION_CERTIFICATE_RENEWAL, auditSource, { domain: domain, errorMessage: errorMessage });
                     mailer.certificateRenewed(domain, errorMessage);
 
                     if (error) {
-                        debug('autoRenew: could not renew cert for %s because %s', domain, error);
+                        debug('renewAll: could not renew cert for %s because %s', domain, error);
 
                         // check if we should fallback if we expire in the coming day
                         if (!isExpiringSync(certFilePath, 24 * 1)) return iteratorCallback();
 
-                        debug('autoRenew: using fallback certs for %s since it expires soon', domain, error);
+                        debug('renewAll: using fallback certs for %s since it expires soon', domain, error);
 
                         certFilePath = 'cert/host.cert';
                         keyFilePath = 'cert/host.key';
                     } else {
-                        debug('autoRenew: certificate for %s renewed', domain);
+                        debug('renewAll: certificate for %s renewed', domain);
                     }
 
                     // reconfigure and reload nginx. this is required for the case where we got a renewed cert after fallback

src/clientdb.js

@@ -5,6 +5,7 @@
 exports = module.exports = {
     get: get,
     getAll: getAll,
+    getAllWithTokenCount: getAllWithTokenCount,
     getAllWithTokenCountByIdentifier: getAllWithTokenCountByIdentifier,
     add: add,
     del: del,
@@ -14,13 +15,7 @@ exports = module.exports = {
     delByAppId: delByAppId,
     delByAppIdAndType: delByAppIdAndType,
 
-    _clear: clear,
-
-    TYPE_EXTERNAL: 'external',
-    TYPE_OAUTH: 'addon-oauth',
-    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
-    TYPE_PROXY: 'addon-proxy',
-    TYPE_ADMIN: 'admin'
+    _clear: clear
 };
 
 var assert = require('assert'),
@@ -52,14 +47,24 @@ function getAll(callback) {
     });
 }
 
+function getAllWithTokenCount(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId GROUP BY clients.id', [], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
 function getAllWithTokenCountByIdentifier(identifier, callback) {
     assert.strictEqual(typeof identifier, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     database.query('SELECT ' + CLIENTS_FIELDS_PREFIXED + ',COUNT(tokens.clientId) AS tokenCount FROM clients LEFT OUTER JOIN tokens ON clients.id=tokens.clientId WHERE tokens.identifier=? GROUP BY clients.id', [ identifier ], function (error, results) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
-        callback(null, results);
 
+        callback(null, results);
     });
 }
 
@@ -71,7 +76,7 @@ function getByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -84,7 +89,7 @@ function getByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null, result[0]);
+        callback(null, result[0]);
     });
 }
 
@@ -127,7 +132,7 @@ function delByAppId(appId, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
@@ -140,17 +145,17 @@ function delByAppIdAndType(appId, type, callback) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 
-        return callback(null);
+        callback(null);
     });
 }
 
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('DELETE FROM clients WHERE appId!="webadmin"', function (error) {
+    database.query('DELETE FROM clients WHERE id!="cid-webadmin" AND id!="cid-sdk" AND id!="cid-cli"', function (error) {
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
-        return callback(null);
+        callback(null);
     });
 }
 

src/clients.js

@@ -6,16 +6,32 @@ exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllWithDetailsByUserId: getAllWithDetailsByUserId,
+    getAll: getAll,
+    getByAppIdAndType: getByAppIdAndType,
     getClientTokensByUserId: getClientTokensByUserId,
     delClientTokensByUserId: delClientTokensByUserId,
+    delByAppIdAndType: delByAppIdAndType,
+    addClientTokenByUserId: addClientTokenByUserId,
+    delToken: delToken,
 
+    // keep this in sync with start.sh ADMIN_SCOPES that generates the cid-webadmin
     SCOPE_APPS: 'apps',
     SCOPE_DEVELOPER: 'developer',
     SCOPE_PROFILE: 'profile',
-    SCOPE_ROOT: 'root',
+    SCOPE_CLOUDRON: 'cloudron',
     SCOPE_SETTINGS: 'settings',
-    SCOPE_USERS: 'users'
+    SCOPE_USERS: 'users',
+
+    // roles are handled just like the above scopes, they are parallel to scopes
+    // scopes enclose API groups, roles specify the usage role
+    SCOPE_ROLE_SDK: 'roleSdk',
+
+    // client type enums
+    TYPE_EXTERNAL: 'external',
+    TYPE_BUILT_IN: 'built-in',
+    TYPE_OAUTH: 'addon-oauth',
+    TYPE_SIMPLE_AUTH: 'addon-simpleauth',
+    TYPE_PROXY: 'addon-proxy'
 };
 
 var assert = require('assert'),
@@ -23,7 +39,6 @@ var assert = require('assert'),
     hat = require('hat'),
     appdb = require('./appdb.js'),
     tokendb = require('./tokendb.js'),
-    constants = require('./constants.js'),
     async = require('async'),
     clientdb = require('./clientdb.js'),
     DatabaseError = require('./databaseerror.js'),
@@ -50,6 +65,10 @@ function ClientsError(reason, errorOrMessage) {
 util.inherits(ClientsError, Error);
 ClientsError.INVALID_SCOPE = 'Invalid scope';
 ClientsError.INVALID_CLIENT = 'Invalid client';
+ClientsError.INVALID_TOKEN = 'Invalid token';
+ClientsError.NOT_FOUND = 'Not found';
+ClientsError.INTERNAL_ERROR = 'Internal Error';
+ClientsError.NOT_ALLOWED = 'Not allowed to remove this client';
 
 function validateScope(scope) {
     assert.strictEqual(typeof scope, 'string');
@@ -58,16 +77,17 @@ function validateScope(scope) {
         exports.SCOPE_APPS,
         exports.SCOPE_DEVELOPER,
         exports.SCOPE_PROFILE,
-        exports.SCOPE_ROOT,
+        exports.SCOPE_CLOUDRON,
         exports.SCOPE_SETTINGS,
-        exports.SCOPE_USERS
+        exports.SCOPE_USERS,
+        '*',    // includes all scopes, but not roles
+        exports.SCOPE_ROLE_SDK
     ];
 
-    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE);
-    if (scope === '*') return null;
+    if (scope === '') return new ClientsError(ClientsError.INVALID_SCOPE, 'Empty scope not allowed');
 
     var allValid = scope.split(',').every(function (s) { return VALID_SCOPES.indexOf(s) !== -1; });
-    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE);
+    if (!allValid) return new ClientsError(ClientsError.INVALID_SCOPE, 'Invalid scope. Available scopes are ' + VALID_SCOPES.join(', '));
 
     return null;
 }
@@ -79,11 +99,14 @@ function add(appId, type, redirectURI, scope, callback) {
     assert.strictEqual(typeof scope, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    // allow whitespace
+    scope = scope.split(',').map(function (s) { return s.trim(); }).join(',');
+
     var error = validateScope(scope);
     if (error) return callback(error);
 
     var id = 'cid-' + uuid.v4();
-    var clientSecret = hat(256);
+    var clientSecret = hat(8 * 128);
 
     clientdb.add(id, appId, type, clientSecret, redirectURI, scope, function (error) {
         if (error) return callback(error);
@@ -106,6 +129,7 @@ function get(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.get(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
@@ -116,24 +140,24 @@ function del(id, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     clientdb.del(id, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
         if (error) return callback(error);
         callback(null, result);
     });
 }
 
-function getAllWithDetailsByUserId(userId, callback) {
-    assert.strictEqual(typeof userId, 'string');
+function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    clientdb.getAllWithTokenCountByIdentifier(tokendb.PREFIX_USER + userId, function (error, results) {
+    clientdb.getAll(function (error, results) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, []);
         if (error) return callback(error);
 
         var tmp = [];
         async.each(results, function (record, callback) {
-            if (record.type === clientdb.TYPE_ADMIN) {
-                record.name = constants.ADMIN_NAME;
-                record.location = constants.ADMIN_LOCATION;
+            if (record.type === exports.TYPE_EXTERNAL || record.type === exports.TYPE_BUILT_IN) {
+                // the appId in this case holds the name
+                record.name = record.appId;
 
                 tmp.push(record);
 
@@ -142,14 +166,13 @@ function getAllWithDetailsByUserId(userId, callback) {
 
             appdb.get(record.appId, function (error, result) {
                 if (error) {
-                    console.error('Failed to get app details for oauth client', result, error);
+                    console.error('Failed to get app details for oauth client', record.appId, error);
                     return callback(null);  // ignore error so we continue listing clients
                 }
 
-                if (record.type === clientdb.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
-                if (record.type === clientdb.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
-                if (record.type === clientdb.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
-                if (record.type === clientdb.TYPE_EXTERNAL) record.name = result.manifest.title + ' external';
+                if (record.type === exports.TYPE_PROXY) record.name = result.manifest.title + ' Website Proxy';
+                if (record.type === exports.TYPE_OAUTH) record.name = result.manifest.title + ' OAuth';
+                if (record.type === exports.TYPE_SIMPLE_AUTH) record.name = result.manifest.title + ' Simple Auth';
 
                 record.location = result.location;
 
@@ -164,15 +187,27 @@ function getAllWithDetailsByUserId(userId, callback) {
     });
 }
 
+function getByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.getByAppIdAndType(appId, type, function (error, result) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null, result);
+    });
+}
+
 function getClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof clientId, 'string');
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.getByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error, result) {
+    tokendb.getByIdentifierAndClientId(userId, clientId, function (error, result) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null, []);
             });
@@ -188,10 +223,10 @@ function delClientTokensByUserId(clientId, userId, callback) {
     assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    tokendb.delByIdentifierAndClientId(tokendb.PREFIX_USER + userId, clientId, function (error) {
+    tokendb.delByIdentifierAndClientId(userId, clientId, function (error) {
         if (error && error.reason === DatabaseError.NOT_FOUND) {
             // this can mean either that there are no tokens or the clientId is actually unknown
-            clientdb.get(clientId, function (error/*, result*/) {
+            get(clientId, function (error/*, result*/) {
                 if (error) return callback(error);
                 callback(null);
             });
@@ -201,3 +236,57 @@ function delClientTokensByUserId(clientId, userId, callback) {
         callback(null);
     });
 }
+
+function delByAppIdAndType(appId, type, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    clientdb.delByAppIdAndType(appId, type, function (error) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.NOT_FOUND, 'No such client'));
+        if (error) return callback(error);
+        callback(null);
+    });
+}
+
+function addClientTokenByUserId(clientId, userId, expiresAt, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof expiresAt, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        var token = tokendb.generateToken();
+
+        tokendb.add(token, userId, result.id, expiresAt, result.scope, function (error) {
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null, {
+                accessToken: token,
+                identifier: userId,
+                clientId: result.id,
+                scope: result.id,
+                expires: expiresAt
+            });
+        });
+    });
+}
+
+function delToken(clientId, tokenId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof tokenId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    get(clientId, function (error, result) {
+        if (error) return callback(error);
+
+        tokendb.del(tokenId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new ClientsError(ClientsError.INVALID_TOKEN, 'Invalid token'));
+            if (error) return callback(new ClientsError(ClientsError.INTERNAL_ERROR, error));
+
+            callback(null);
+        });
+    });
+}

src/cloudron.js

@@ -12,7 +12,6 @@ exports = module.exports = {
     sendHeartbeat: sendHeartbeat,
 
     updateToLatest: updateToLatest,
-    update: update,
     reboot: reboot,
     retire: retire,
 
@@ -31,8 +30,9 @@ var apps = require('./apps.js'),
     assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
+    constants = require('./constants.js'),
     debug = require('debug')('box:cloudron'),
     df = require('node-df'),
     eventlog = require('./eventlog.js'),
@@ -53,9 +53,8 @@ var apps = require('./apps.js'),
     updateChecker = require('./updatechecker.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    userdb = require('./userdb.js'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    user = require('./user.js'),
+    util = require('util');
 
 var REBOOT_CMD = path.join(__dirname, 'scripts/reboot.sh'),
     INSTALLER_UPDATE_URL = 'http://127.0.0.1:2020/api/v1/installer/update',
@@ -91,10 +90,6 @@ CloudronError.BAD_FIELD = 'Field error';
 CloudronError.INTERNAL_ERROR = 'Internal Error';
 CloudronError.EXTERNAL_ERROR = 'External Error';
 CloudronError.ALREADY_PROVISIONED = 'Already Provisioned';
-CloudronError.BAD_USERNAME = 'Bad username';
-CloudronError.BAD_EMAIL = 'Bad email';
-CloudronError.BAD_PASSWORD = 'Bad password';
-CloudronError.BAD_NAME = 'Bad name';
 CloudronError.BAD_STATE = 'Bad state';
 CloudronError.ALREADY_UPTODATE = 'No Update Available';
 CloudronError.NOT_FOUND = 'Not found';
@@ -231,19 +226,17 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 
     user.createOwner(username, password, email, displayName, auditSource, function (error, userObject) {
         if (error && error.reason === UserError.ALREADY_EXISTS) return callback(new CloudronError(CloudronError.ALREADY_PROVISIONED));
-        if (error && error.reason === UserError.BAD_USERNAME) return callback(new CloudronError(CloudronError.BAD_USERNAME));
-        if (error && error.reason === UserError.BAD_PASSWORD) return callback(new CloudronError(CloudronError.BAD_PASSWORD));
-        if (error && error.reason === UserError.BAD_EMAIL) return callback(new CloudronError(CloudronError.BAD_EMAIL));
+        if (error && error.reason === UserError.BAD_FIELD) return callback(new CloudronError(CloudronError.BAD_FIELD, error.message));
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+        clients.get('cid-webadmin', function (error, result) {
             if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
             // Also generate a token so the admin creation can also act as a login
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + userObject.id, result.id, expires, '*', function (error) {
+            tokendb.add(token, userObject.id, result.id, expires, '*', function (error) {
                 if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
                 // EE API is sync. do not keep the REST API reponse waiting
@@ -260,7 +253,7 @@ function activate(username, password, email, displayName, ip, auditSource, callb
 function getStatus(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.count(function (error, count) {
+    user.count(function (error, count) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
         settings.getCloudronName(function (error, cloudronName) {
@@ -444,16 +437,19 @@ function addDnsRecords() {
     sysinfo.getIp(function (error, ip) {
         if (error) return callback(new CloudronError(CloudronError.INTERNAL_ERROR, error));
 
-        var webadminRecord = { subdomain: 'my', type: 'A', values: [ ip ] };
+        var webadminRecord = { subdomain: constants.ADMIN_LOCATION, type: 'A', values: [ ip ] };
         // t=s limits the domainkey to this domain and not it's subdomains
         var dkimRecord = { subdomain: DKIM_SELECTOR + '._domainkey', type: 'TXT', values: [ '"v=DKIM1; t=s; p=' + dkimKey + '"' ] };
         // DMARC requires special setup if report email id is in different domain
         var dmarcRecord = { subdomain: '_dmarc', type: 'TXT', values: [ '"v=DMARC1; p=none; pct=100; rua=mailto:' + DMARC_REPORT_EMAIL + '; ruf=' + DMARC_REPORT_EMAIL + '"' ] };
 
+        var mxRecord = { subdomain: '', type: 'MX', values: [ '10 ' + config.mailFqdn() + '.' ] };
+
         var records = [ ];
         if (config.isCustomDomain()) {
             records.push(webadminRecord);
             records.push(dkimRecord);
+            records.push(mxRecord);
         } else {
             // for custom domains, we show a nakeddomain.html page
             var nakedDomainRecord = { subdomain: '', type: 'A', values: [ ip ] };
@@ -462,6 +458,7 @@ function addDnsRecords() {
             records.push(webadminRecord);
             records.push(dkimRecord);
             records.push(dmarcRecord);
+            records.push(mxRecord);
         }
 
         debug('addDnsRecords: %j', records);
@@ -497,8 +494,9 @@ function reboot(callback) {
     shell.sudo('reboot', [ REBOOT_CMD ], callback);
 }
 
-function update(boxUpdateInfo, callback) {
+function update(boxUpdateInfo, auditSource, callback) {
     assert.strictEqual(typeof boxUpdateInfo, 'object');
+    assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     if (!boxUpdateInfo) return callback(null);
@@ -506,6 +504,8 @@ function update(boxUpdateInfo, callback) {
     var error = locker.lock(locker.OP_BOX_UPDATE);
     if (error) return callback(new CloudronError(CloudronError.BAD_STATE, error.message));
 
+    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
+
     // ensure tools can 'wait' on progress
     progress.set(progress.UPDATE, 0, 'Starting');
 
@@ -544,9 +544,7 @@ function updateToLatest(auditSource, callback) {
     var boxUpdateInfo = updateChecker.getUpdateInfo().box;
     if (!boxUpdateInfo) return callback(new CloudronError(CloudronError.ALREADY_UPTODATE, 'No update available'));
 
-    eventlog.add(eventlog.ACTION_UPDATE, auditSource, { boxUpdateInfo: boxUpdateInfo });
-
-    update(boxUpdateInfo, callback);
+    update(boxUpdateInfo, auditSource, callback);
 }
 
 function doShortCircuitUpdate(boxUpdateInfo, callback) {
@@ -656,23 +654,16 @@ function installAppBundle(callback) {
     }
 
     async.eachSeries(bundle, function (appInfo, iteratorCallback) {
-        var appstoreId = appInfo.appstoreId;
-        var parts = appstoreId.split('@');
+        debug('autoInstall: installing %s at %s', appInfo.appstoreId, appInfo.location);
 
-        var url = config.apiServerOrigin() + '/api/v1/apps/' + parts[0] + (parts[1] ? '/versions/' + parts[1] : '');
+        var data = {
+            appStoreId: appInfo.appstoreId,
+            location: appInfo.location,
+            portBindings: appInfo.portBindings || null,
+            accessRestriction: appInfo.accessRestriction || null,
+        };
 
-        superagent.get(url).end(function (error, result) {
-            if (error && !error.response) return iteratorCallback(new Error('Network error: ' + error.message));
-
-            if (result.statusCode !== 200) return iteratorCallback(util.format('Failed to get app info from store.', result.statusCode, result.text));
-
-            debug('autoInstall: installing %s at %s', appstoreId, appInfo.location);
-
-            apps.install(uuid.v4(), appstoreId, result.body.manifest, appInfo.location,
-                appInfo.portBindings || null, appInfo.accessRestriction || null,
-                null /* icon */, null /* cert */, null /* key */, 0 /* default mem limit */,
-                null /* altDomain */, { userId: null, username: 'autoinstaller' }, iteratorCallback);
-        });
+        apps.install(data, { userId: null, username: 'autoinstaller' }, iteratorCallback);
     }, function (error) {
         if (error) debug('autoInstallApps: ', error);
 

src/config.js

@@ -28,9 +28,9 @@ exports = module.exports = {
     internalAdminOrigin: internalAdminOrigin,
     sysadminOrigin: sysadminOrigin, // caas routes
     adminFqdn: adminFqdn,
+    mailFqdn: mailFqdn,
     appFqdn: appFqdn,
     zoneName: zoneName,
-    adminEmail: adminEmail,
 
     isDev: isDev,
 
@@ -73,11 +73,11 @@ function initConfig() {
     data.fqdn = 'localhost';
 
     data.token = null;
-    data.adminEmail = null;
     data.boxVersionsUrl = null;
     data.version = null;
     data.isCustomDomain = false;
     data.webServerOrigin = null;
+    data.smtpPort = 2525; // // this value comes from mail container
     data.sysadminPort = 3001;
     data.ldapPort = 3002;
     data.oauthProxyPort = 3003;
@@ -100,7 +100,6 @@ function initConfig() {
             name: 'boxtest'
         };
         data.token = 'APPSTORE_TOKEN';
-        data.adminEmail = 'test@cloudron.foo';
     } else {
         assert(false, 'Unknown environment. This should not happen!');
     }
@@ -139,10 +138,6 @@ function get(key) {
     return safe.query(data, key);
 }
 
-function adminEmail() {
-    return get('adminEmail');
-}
-
 function apiServerOrigin() {
     return get('apiServerOrigin');
 }
@@ -167,6 +162,10 @@ function adminFqdn() {
     return appFqdn(constants.ADMIN_LOCATION);
 }
 
+function mailFqdn() {
+    return appFqdn(constants.MAIL_LOCATION);
+}
+
 function adminOrigin() {
     return 'https://' + appFqdn(constants.ADMIN_LOCATION);
 }

src/constants.js

@@ -6,6 +6,8 @@ exports = module.exports = {
     API_LOCATION: 'api', // this is unused but reserved for future use (#403)
     SMTP_LOCATION: 'smtp',
     IMAP_LOCATION: 'imap',
+    MAIL_LOCATION: 'my', // not a typo! should be same as admin location until we figure out certificates
+    POSTMAN_LOCATION: 'postman', // used in dovecot bounces
 
     ADMIN_NAME: 'Settings',
 

src/cron.js

@@ -30,6 +30,7 @@ var gAutoupdaterJob = null,
     gCheckDiskSpaceJob = null;
 
 var NOOP_CALLBACK = function (error) { if (error) console.error(error); };
+var AUDIT_SOURCE = { userId: null, username: 'cron' };
 
 // cron format
 // Seconds: 0-59
@@ -66,7 +67,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gBackupJob) gBackupJob.stop();
         gBackupJob = new CronJob({
             cronTime: '00 00 */4 * * *', // every 4 hours
-            onTick: backups.ensureBackup,
+            onTick: backups.ensureBackup.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -122,7 +123,7 @@ function recreateJobs(unusedTimeZone, callback) {
         if (gCertificateRenewJob) gCertificateRenewJob.stop();
         gCertificateRenewJob = new CronJob({
             cronTime: '00 00 */12 * * *', // every 12 hours
-            onTick: certificates.autoRenew,
+            onTick: certificates.renewAll.bind(null, AUDIT_SOURCE, NOOP_CALLBACK),
             start: true,
             timeZone: allSettings[settings.TIME_ZONE_KEY]
         });
@@ -154,10 +155,10 @@ function autoupdatePatternChanged(pattern) {
             var updateInfo = updateChecker.getUpdateInfo();
             if (updateInfo.box) {
                 debug('Starting autoupdate to %j', updateInfo.box);
-                cloudron.update(updateInfo.box, NOOP_CALLBACK);
+                cloudron.updateToLatest(AUDIT_SOURCE, NOOP_CALLBACK);
             } else if (updateInfo.apps) {
                 debug('Starting app update to %j', updateInfo.apps);
-                apps.autoupdateApps(updateInfo.apps, NOOP_CALLBACK);
+                apps.updateApps(updateInfo.apps, AUDIT_SOURCE, NOOP_CALLBACK);
             } else {
                 debug('No auto updates available');
             }

src/database.js

@@ -1,5 +1,3 @@
-/* jslint node: true */
-
 'use strict';
 
 exports = module.exports = {
@@ -122,7 +120,8 @@ function clear(callback) {
         require('./groupdb.js')._clear,
         require('./userdb.js')._clear,
         require('./settingsdb.js')._clear,
-        require('./eventlogdb.js')._clear
+        require('./eventlogdb.js')._clear,
+        require('./mailboxdb.js')._clear
     ], callback);
 }
 

src/developer.js

@@ -5,7 +5,7 @@
 exports = module.exports = {
     DeveloperError: DeveloperError,
 
-    enabled: enabled,
+    isEnabled: isEnabled,
     setEnabled: setEnabled,
     issueDeveloperToken: issueDeveloperToken,
     getNonApprovedApps: getNonApprovedApps
@@ -13,6 +13,7 @@ exports = module.exports = {
 
 var assert = require('assert'),
     config = require('./config.js'),
+    clients = require('./clients.js'),
     debug = require('debug')('box:developer'),
     eventlog = require('./eventlog.js'),
     tokendb = require('./tokendb.js'),
@@ -42,7 +43,7 @@ util.inherits(DeveloperError, Error);
 DeveloperError.INTERNAL_ERROR = 'Internal Error';
 DeveloperError.EXTERNAL_ERROR = 'External Error';
 
-function enabled(callback) {
+function isEnabled(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     settings.getDeveloperMode(function (error, enabled) {
@@ -72,13 +73,14 @@ function issueDeveloperToken(user, auditSource, callback) {
 
     var token = tokendb.generateToken();
     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
+    var scopes = '*,' + clients.SCOPE_ROLE_SDK;
 
-    tokendb.add(token, tokendb.PREFIX_DEV + user.id, '', expiresAt, 'developer,apps,settings,users,profile', function (error) {
+    tokendb.add(token, user.id, 'cid-cli', expiresAt, scopes, function (error) {
         if (error) return callback(new DeveloperError(DeveloperError.INTERNAL_ERROR, error));
 
         eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource, { authType: 'cli', userId: user.id, username: user.username });
 
-        callback(null, { token: token, expiresAt: expiresAt });
+        callback(null, { token: token, expiresAt: new Date(expiresAt).toISOString() });
     });
 }
 

src/docker.js

@@ -133,7 +133,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var docker = exports.connection,
-        isAppContainer = !cmd;
+        isAppContainer = !cmd; // non app-containers are like scheduler containers
 
     var manifest = app.manifest;
     var developmentMode = !!manifest.developmentMode;
@@ -172,18 +172,16 @@ function createSubcontainer(app, name, cmd, options, callback) {
     // developerMode does not restrict memory usage
     memoryLimit = developmentMode ? 0 : memoryLimit;
 
-    // for subcontainers, this should ideally be false. but docker does not allow network sharing if the app container is not running
-    // this means cloudron exec does not work
-    var isolatedNetworkNs = true;
-
     addons.getEnvironment(app, function (error, addonEnv) {
         if (error) return callback(new Error('Error getting addon environment : ' + error));
 
+        // do no set hostname of containers to location as it might conflict with addons names. for example, an app installed in mail
+        // location may not reach mail container anymore by DNS. We cannot set hostname to fqdn either as that sets up the dns
+        // name to look up the internal docker ip. this makes curl from within container fail
+        // Note that Hostname has no effect on DNS. We have to use the --net-alias for dns.
+        // Hostname cannot be set with container NetworkMode
         var containerOptions = {
             name: name, // used for filtering logs
-            // do _not_ set hostname to app fqdn. doing so sets up the dns name to look up the internal docker ip. this makes curl from within container fail
-            // for subcontainers, this should not be set because we already share the network namespace with app container
-            Hostname: isolatedNetworkNs ? (semver.gte(targetBoxVersion(app.manifest), '0.0.77') ? app.location : config.appFqdn(app.location)) : null,
             Tty: isAppContainer,
             Image: app.manifest.dockerImage,
             Cmd: (isAppContainer && developmentMode) ? [ '/bin/bash', '-c', 'echo "Development mode. Use cloudron exec to debug. Sleeping" && sleep infinity' ] : cmd,
@@ -211,8 +209,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 },
                 CpuShares: 512, // relative to 1024 for system processes
                 VolumesFrom: isAppContainer ? null : [ app.containerId + ":rw" ],
-                NetworkMode: isolatedNetworkNs ? 'default' : ('container:' + app.containerId), // share network namespace with parent
-                Links: isolatedNetworkNs ? addons.getLinksSync(app, app.manifest.addons) : null, // links is redundant with --net=container
+                NetworkMode: isAppContainer ? 'cloudron' : ('container:' + app.containerId), // share network namespace with parent
                 SecurityOpt: config.CLOUDRON ? [ "apparmor:docker-cloudron-app" ] : null // profile available only on cloudron
             }
         };
@@ -375,10 +372,10 @@ function getContainerIdByIp(ip, callback) {
 
         var bridge;
         result.forEach(function (n) {
-            if (n.Name === 'bridge') bridge = n;
+            if (n.Name === 'cloudron') bridge = n;
         });
 
-        if (!bridge) return callback(new Error('Unable to find the bridge network'));
+        if (!bridge) return callback(new Error('Unable to find the cloudron network'));
 
         var containerId;
         for (var id in bridge.Containers) {

src/eventlog.js

@@ -9,6 +9,7 @@ exports = module.exports = {
 
     // keep in sync with webadmin index.js filter
     ACTION_ACTIVATE: 'cloudron.activate',
+    ACTION_APP_CLONE: 'app.clone',
     ACTION_APP_CONFIGURE: 'app.configure',
     ACTION_APP_INSTALL: 'app.install',
     ACTION_APP_RESTORE: 'app.restore',

src/groupdb.js

@@ -4,6 +4,7 @@ exports = module.exports = {
     get: get,
     getWithMembers: getWithMembers,
     getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
     add: add,
     del: del,
     count: count,
@@ -65,6 +66,19 @@ function getAll(callback) {
     });
 }
 
+function getAllWithMembers(callback) {
+    database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
+                    ' FROM groups LEFT OUTER JOIN groupMembers ON groups.id = groupMembers.groupId ' +
+                    ' GROUP BY groups.id', function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
+
+        callback(null, results);
+    });
+}
+
 function add(id, name, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof name, 'string');

src/groups.js

@@ -10,6 +10,7 @@ exports = module.exports = {
     get: get,
     getWithMembers: getWithMembers,
     getAll: getAll,
+    getAllWithMembers: getAllWithMembers,
 
     getMembers: getMembers,
     addMember: addMember,
@@ -51,7 +52,7 @@ util.inherits(GroupError, Error);
 GroupError.INTERNAL_ERROR = 'Internal Error';
 GroupError.ALREADY_EXISTS = 'Already Exists';
 GroupError.NOT_FOUND = 'Not Found';
-GroupError.BAD_NAME = 'Bad name';
+GroupError.BAD_FIELD = 'Field error';
 GroupError.NOT_EMPTY = 'Not Empty';
 GroupError.NOT_ALLOWED = 'Not Allowed';
 
@@ -59,12 +60,12 @@ function validateGroupname(name) {
     assert.strictEqual(typeof name, 'string');
     var RESERVED = [ 'admins', 'users' ]; // ldap code uses 'users' pseudo group
 
-    if (name.length <= 2) return new GroupError(GroupError.BAD_NAME, 'name must be atleast 2 chars');
-    if (name.length >= 200) return new GroupError(GroupError.BAD_NAME, 'name too long');
+    if (name.length <= 2) return new GroupError(GroupError.BAD_FIELD, 'name must be atleast 2 chars');
+    if (name.length >= 200) return new GroupError(GroupError.BAD_FIELD, 'name too long');
 
-    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_NAME, 'name can only have A-Za-z0-9_-');
+    if (!/^[A-Za-z0-9_-]*$/.test(name)) return new GroupError(GroupError.BAD_FIELD, 'name can only have A-Za-z0-9_-');
 
-    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_NAME, 'name is reserved');
+    if (RESERVED.indexOf(name) !== -1) return new GroupError(GroupError.BAD_FIELD, 'name is reserved');
 
     return null;
 }
@@ -133,6 +134,16 @@ function getAll(callback) {
     });
 }
 
+function getAllWithMembers(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getAllWithMembers(function (error, result) {
+        if (error) return callback(new GroupError(GroupError.INTERNAL_ERROR, error));
+
+        return callback(null, result);
+    });
+}
+
 function getMembers(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof callback, 'function');

src/infra_version.js

@@ -0,0 +1,21 @@
+'use strict';
+
+// WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
+// These constants are used in the installer script as well
+// Do not require anything here!
+
+exports = module.exports = {
+    'version': 37,
+
+    'baseImage': 'cloudron/base:0.8.1',
+
+    'images': {
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:0.12.0' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:0.11.0' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:0.10.0' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:0.9.0' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:0.14.0' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:0.9.0' }
+    }
+};
+

src/ldap.js

@@ -12,7 +12,9 @@ var assert = require('assert'),
     eventlog = require('./eventlog.js'),
     user = require('./user.js'),
     UserError = user.UserError,
-    ldap = require('ldapjs');
+    ldap = require('ldapjs'),
+    mailboxes = require('./mailboxes.js'),
+    MailboxError = mailboxes.MailboxError;
 
 var gServer = null;
 
@@ -35,8 +37,204 @@ function getAppByRequest(req, callback) {
     if (sourceIp.split('.').length !== 4) return callback(new ldap.InsufficientAccessRightsError('Missing source identifier'));
 
     apps.getByIpAddress(sourceIp, function (error, app) {
-        // we currently allow access in case we can't find the source app
-        callback(null, app || null);
+        if (error) return callback(new ldap.OperationsError(error.message));
+
+        if (!app) return callback(new ldap.OperationsError('Could not detect app source'));
+
+        callback(null, app);
+    });
+}
+
+function userSearch(req, res, next) {
+    debug('user search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        // send user objects
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
+
+            var groups = [ GROUP_USERS_DN ];
+            if (entry.admin) groups.push(GROUP_ADMINS_DN);
+
+            var displayName = entry.displayName || entry.username;
+            var nameParts = displayName.split(' ');
+            var firstName = nameParts[0];
+            var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['user'],
+                    objectcategory: 'person',
+                    cn: entry.id,
+                    uid: entry.id,
+                    mail: entry.email,
+                    // TODO: check mailboxes before we send this
+                    mailAlternateAddress: entry.username + '@' + config.fqdn(),
+                    displayname: displayName,
+                    givenName: firstName,
+                    username: entry.username,
+                    samaccountname: entry.username,      // to support ActiveDirectory clients
+                    memberof: groups
+                }
+            };
+
+            // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
+            // which is required to have atleast one character if present
+            if (lastName.length !== 0) obj.attributes.sn = lastName;
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function groupSearch(req, res, next) {
+    debug('group search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    user.list(function (error, result){
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        var groups = [{
+            name: 'users',
+            admin: false
+        }, {
+            name: 'admins',
+            admin: true
+        }];
+
+        groups.forEach(function (group) {
+            var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
+            var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
+
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['group'],
+                    cn: group.name,
+                    memberuid: members.map(function(entry) { return entry.id; })
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function mailboxSearch(req, res, next) {
+    debug('mailbox search: dn %s, scope %s, filter %s (from %s)', req.dn.toString(), req.scope, req.filter.toString(), req.connection.ldap.id);
+
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new ldap.OperationsError(error.toString()));
+
+        result.forEach(function (entry) {
+            var dn = ldap.parseDN('cn=' + entry.name + ',ou=mailboxes,dc=cloudron');
+
+            // TODO: send aliases
+            var obj = {
+                dn: dn.toString(),
+                attributes: {
+                    objectclass: ['mailbox'],
+                    objectcategory: 'mailbox',
+                    cn: entry.name,
+                    uid: entry.name,
+                    mail: entry.name + '@' + config.fqdn()
+                }
+            };
+
+            // ensure all filter values are also lowercase
+            var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
+
+            if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
+                res.send(obj);
+            }
+        });
+
+        res.end();
+    });
+}
+
+function authenticateUser(req, res, next) {
+    debug('user bind: %s (from %s)', req.dn.toString(), req.connection.ldap.id);
+
+    // extract the common name which might have different attribute names
+    var attributeName = Object.keys(req.dn.rdns[0])[0];
+    var commonName = req.dn.rdns[0][attributeName];
+    if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+    var api;
+    if (attributeName === 'mail') {
+        api = user.verifyWithEmail;
+    } else if (commonName.indexOf('@') !== -1) { // if mail is specified, enforce mail check
+        var parts = commonName.split('@');
+        if (parts[1] === config.fqdn()) { // internal email, verify with username
+            commonName = parts[0];
+            api = user.verifyWithUsername;
+        } else { // external email
+            api = user.verifyWithEmail;
+        }
+    } else if (commonName.indexOf('uid-') === 0) {
+        api = user.verify;
+    } else {
+        api = user.verifyWithUsername;
+    }
+
+    api(commonName, req.credentials || '', function (error, user) {
+        if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        req.user = user;
+
+        next();
+    });
+}
+
+function authorizeUserForApp(req, res, next) {
+    assert(req.user);
+
+    getAppByRequest(req, function (error, app) {
+        if (error) return next(error);
+
+        apps.hasAccessTo(app, req.user, function (error, result) {
+            if (error) return next(new ldap.OperationsError(error.toString()));
+
+            // we return no such object, to avoid leakage of a users existence
+            if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+
+            eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: req.user.id });
+
+            res.end();
+        });
+    });
+}
+
+function authorizeUserForMailbox(req, res, next) {
+    assert(req.user);
+
+    // We simply authorize the user to access a mailbox by his own name
+    mailboxes.get(req.user.username, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
+        if (error) return next(new ldap.OperationsError(error.message));
+
+        eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', mailboxId: req.user.username }, { userId: req.user.username });
+
+        res.end();
     });
 }
 
@@ -45,155 +243,27 @@ function start(callback) {
 
     gServer = ldap.createServer({ log: gLogger });
 
-    gServer.search('ou=users,dc=cloudron', function (req, res, next) {
-        debug('user search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
+    gServer.search('ou=users,dc=cloudron', userSearch);
+    gServer.search('ou=groups,dc=cloudron', groupSearch);
+    gServer.bind('ou=users,dc=cloudron', authenticateUser, authorizeUserForApp);
 
-        user.list(function (error, result) {
-            if (error) return next(new ldap.OperationsError(error.toString()));
+    gServer.search('ou=mailboxes,dc=cloudron', mailboxSearch);
+    gServer.bind('ou=mailboxes,dc=cloudron', authenticateUser, authorizeUserForMailbox);
 
-            // send user objects
-            result.forEach(function (entry) {
-                var dn = ldap.parseDN('cn=' + entry.id + ',ou=users,dc=cloudron');
-
-                var groups = [ GROUP_USERS_DN ];
-                if (entry.admin) groups.push(GROUP_ADMINS_DN);
-
-                var displayName = entry.displayName || entry.username;
-                var nameParts = displayName.split(' ');
-                var firstName = nameParts[0];
-                var lastName = nameParts.length > 1  ? nameParts[nameParts.length - 1] : ''; // choose last part, if it exists
-
-                var obj = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['user'],
-                        objectcategory: 'person',
-                        cn: entry.id,
-                        uid: entry.id,
-                        mail: entry.email,
-                        displayname: displayName,
-                        givenName: firstName,
-                        username: entry.username,
-                        samaccountname: entry.username,      // to support ActiveDirectory clients
-                        memberof: groups
-                    }
-                };
-
-                // http://www.zytrax.com/books/ldap/ape/core-schema.html#sn has 'name' as SUP which is a DirectoryString
-                // which is required to have atleast one character if present
-                if (lastName.length !== 0) obj.attributes.sn = lastName;
-
-                // ensure all filter values are also lowercase
-                var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                    res.send(obj);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    gServer.search('ou=groups,dc=cloudron', function (req, res, next) {
-        debug('group search: dn %s, scope %s, filter %s', req.dn.toString(), req.scope, req.filter.toString());
-
-        user.list(function (error, result){
-            if (error) return next(new ldap.OperationsError(error.toString()));
-
-            var groups = [{
-                name: 'users',
-                admin: false
-            }, {
-                name: 'admins',
-                admin: true
-            }];
-
-            groups.forEach(function (group) {
-                var dn = ldap.parseDN('cn=' + group.name + ',ou=groups,dc=cloudron');
-                var members = group.admin ? result.filter(function (entry) { return entry.admin; }) : result;
-
-                var obj = {
-                    dn: dn.toString(),
-                    attributes: {
-                        objectclass: ['group'],
-                        cn: group.name,
-                        memberuid: members.map(function(entry) { return entry.id; })
-                    }
-                };
-
-                // ensure all filter values are also lowercase
-                var lowerCaseFilter = ldap.parseFilter(req.filter.toString().toLowerCase());
-
-                if ((req.dn.equals(dn) || req.dn.parentOf(dn)) && lowerCaseFilter.matches(obj.attributes)) {
-                    res.send(obj);
-                }
-            });
-
-            res.end();
-        });
-    });
-
-    // this is the bind for the mail addon to authorize apps
-    gServer.bind('ou=sendmail,dc=cloudron', function(req, res, next) {
-        // TODO: validate password
-        debug('sendmail bind: %s', req.dn.toString()); // note: cn can be email or id
+    // this is the bind for addons (after bind, they might search and authenticate)
+    gServer.bind('ou=addons,dc=cloudron', function(req, res, next) {
+        debug('addons bind: %s', req.dn.toString()); // note: cn can be email or id
         res.end();
     });
 
+    // this is the bind for apps (after bind, they might search and authenticate user)
     gServer.bind('ou=apps,dc=cloudron', function(req, res, next) {
         // TODO: validate password
         debug('application bind: %s', req.dn.toString());
         res.end();
     });
 
-    gServer.bind('ou=users,dc=cloudron', function(req, res, next) {
-        debug('user bind: %s', req.dn.toString());
-
-        // extract the common name which might have different attribute names
-        var attributeName = Object.keys(req.dn.rdns[0])[0];
-        var commonName = req.dn.rdns[0][attributeName];
-        if (!commonName) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-        var api;
-        // if mail is specified, enforce mail check
-        if (commonName.indexOf('@') !== -1 || attributeName === 'mail') {
-            api = user.verifyWithEmail;
-        } else if (commonName.indexOf('uid-') === 0) {
-            api = user.verify;
-        } else {
-            api = user.verifyWithUsername;
-        }
-
-        // TODO this should be done after we verified the app has access to avoid leakage of user existence
-        api(commonName, req.credentials || '', function (error, userObject) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-            if (error && error.reason === UserError.WRONG_PASSWORD) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
-            if (error) return next(new ldap.OperationsError(error));
-
-            getAppByRequest(req, function (error, app) {
-                if (error) return next(error);
-
-                if (!app) {
-                    debug('no app found for this container, allow access');
-                    return res.end();
-                }
-
-                apps.hasAccessTo(app, userObject, function (error, result) {
-                    if (error) return next(new ldap.OperationsError(error.toString()));
-
-                    // we return no such object, to avoid leakage of a users existence
-                    if (!result) return next(new ldap.NoSuchObjectError(req.dn.toString()));
-
-                    eventlog.add(eventlog.ACTION_USER_LOGIN, { authType: 'ldap', appId: app.id }, { userId: userObject.id });
-
-                    res.end();
-                });
-            });
-        });
-    });
-
-    gServer.listen(config.get('ldapPort'), callback);
+    gServer.listen(config.get('ldapPort'), '0.0.0.0', callback);
 }
 
 function stop(callback) {

src/mail_templates/user_added.ejs

@@ -2,7 +2,7 @@
 
 Dear Admin,
 
-User with name <%= user.email %> was added in the Cloudron at <%= fqdn %>.
+User with email <%= user.email %> was added in the Cloudron at <%= fqdn %>.
 
 You are receiving this email because you are an Admin of the Cloudron at <%= fqdn %>.
 

src/mailboxdb.js

@@ -0,0 +1,127 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    getAliases: getAliases,
+    setAliases: setAliases,
+
+    _clear: clear
+};
+
+var assert = require('assert'),
+    database = require('./database.js'),
+    DatabaseError = require('./databaseerror.js'),
+    util = require('util');
+
+var MAILBOX_FIELDS = [ 'name', 'aliasTarget', 'creationTime' ].join(',');
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mailboxes (name) VALUES (?)', [ name ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function clear(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('TRUNCATE TABLE mailboxes', [], function (error) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        callback(null);
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mailboxes WHERE name=? OR aliasTarget = ?', [ name, name ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        callback(null);
+    });
+}
+
+function postProcess(result) {
+    result.aliases = result.aliases ? result.aliases.split(',') : [ ];
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.name=? AND m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (results.length === 0) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        postProcess(results[0]);
+
+        callback(null, results[0]);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    var query = 'SELECT m1.name, m1.creationTime, GROUP_CONCAT(m2.name) AS aliases ' +
+                'FROM mailboxes as m1 ' +
+                'LEFT OUTER JOIN mailboxes as m2 ON m1.name = m2.aliasTarget ' +
+                'WHERE m1.aliasTarget IS NULL ' +
+                'GROUP BY m1.name';
+
+    database.query(query, function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results.forEach(postProcess);
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    // also cleanup the groupMembers table
+    var queries = [];
+    queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ?', args: [ name ] });
+    aliases.forEach(function (alias) {
+        queries.push({ query: 'INSERT INTO mailboxes (name, aliasTarget) VALUES (?, ?)', args: [ alias, name ] });
+    });
+
+    database.transaction(queries, function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error.message));
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        callback(null);
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT name FROM mailboxes WHERE aliasTarget=? ORDER BY name', [ name ], function (error, results) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+
+        results = results.map(function (r) { return r.name; });
+        callback(null, results);
+    });
+}

src/mailboxes.js

@@ -0,0 +1,198 @@
+'use strict';
+
+exports = module.exports = {
+    add: add,
+    del: del,
+    get: get,
+    getAll: getAll,
+    setAliases: setAliases,
+    getAliases: getAliases,
+
+    setupAliases: setupAliases,
+
+    MailboxError: MailboxError
+};
+
+var assert = require('assert'),
+    async = require('async'),
+    DatabaseError = require('./databaseerror.js'),
+    debug = require('debug')('box:mailboxes'),
+    docker = require('./docker.js'),
+    mailboxdb = require('./mailboxdb.js'),
+    util = require('util');
+
+function MailboxError(reason, errorOrMessage) {
+    assert.strictEqual(typeof reason, 'string');
+    assert(errorOrMessage instanceof Error || typeof errorOrMessage === 'string' || typeof errorOrMessage === 'undefined');
+
+    Error.call(this);
+    Error.captureStackTrace(this, this.constructor);
+
+    this.name = this.constructor.name;
+    this.reason = reason;
+    if (typeof errorOrMessage === 'undefined') {
+        this.message = reason;
+    } else if (typeof errorOrMessage === 'string') {
+        this.message = errorOrMessage;
+    } else {
+        this.message = 'Internal error';
+        this.nestedError = errorOrMessage;
+    }
+}
+util.inherits(MailboxError, Error);
+MailboxError.ALREADY_EXISTS = 'already exists';
+MailboxError.BAD_FIELD = 'Field error';
+MailboxError.NOT_FOUND = 'not found';
+MailboxError.INTERNAL_ERROR = 'internal error';
+MailboxError.EXTERNAL_ERROR = 'external error';
+
+function validateName(name) {
+    var RESERVED_NAMES = [ 'no-reply', 'postmaster', 'mailer-daemon' ];
+
+    if (!name.length) return new MailboxError(MailboxError.BAD_FIELD, "name cannot be empty");
+
+    if (name.length < 2) return new MailboxError(MailboxError.BAD_FIELD, 'name too small');
+    if (name.length > 127) return new MailboxError(MailboxError.BAD_FIELD, 'name too long');
+    if (RESERVED_NAMES.indexOf(name) !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'name is reserved');
+
+    if (/[^a-zA-Z0-9.]/.test(name)) return new MailboxError(MailboxError.BAD_FIELD, 'name can only contain alphanumerals and dot');
+
+    if (name.indexOf('.app') !== -1) return new MailboxError(MailboxError.BAD_FIELD, 'alias pattern is reserved for apps');
+
+    return null;
+}
+
+function add(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    name = name.toLowerCase();
+
+    var error = validateName(name);
+    if (error) return callback(error);
+
+    mailboxdb.add(name, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        debug('Added mailbox %s', name);
+
+        var mailbox = {
+            name: name
+        };
+
+        callback(null, mailbox);
+    });
+}
+
+function pushAlias(name, aliases, callback) {
+    if (process.env.BOX_ENV === 'test') return callback();
+
+    var cmd = [ '/addons/mail/service.sh', 'set-alias', name ].concat(aliases);
+
+    debug('pushing alias for %s : %j', name, aliases);
+
+    docker.execContainer('mail', cmd, { }, function (error) {
+        if (error) return callback(new MailboxError(MailboxError.EXTERNAL_ERROR, error));
+
+        callback();
+    });
+}
+
+function del(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    pushAlias(name, [ ], function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.del(name, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+            debug('deleted mailbox %s', name);
+
+            callback();
+        });
+    });
+}
+
+function get(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.get(name, function (error, mailbox) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, mailbox);
+    });
+}
+
+function getAll(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAll(function (error, results) {
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, results);
+    });
+}
+
+function setAliases(name, aliases, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert(util.isArray(aliases));
+    assert.strictEqual(typeof callback, 'function');
+
+    for (var i = 0; i < aliases.length; i++) {
+        aliases[i] = aliases[i].toLowerCase();
+
+        var error = validateName(aliases[i]);
+        if (error) return callback(error);
+    }
+
+    pushAlias(name, aliases, function (error) {
+        if (error) return callback(error);
+
+        mailboxdb.setAliases(name, aliases, function (error) {
+            if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new MailboxError(MailboxError.ALREADY_EXISTS, error.message))
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+            if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+
+            callback(null);
+        });
+    });
+}
+
+function getAliases(name, callback) {
+    assert.strictEqual(typeof name, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    mailboxdb.getAliases(name, function (error, aliases) {
+        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new MailboxError(MailboxError.NOT_FOUND));
+        if (error) return callback(new MailboxError(MailboxError.INTERNAL_ERROR, error));
+
+        callback(null, aliases);
+    });
+}
+
+// push aliases to the mail container on startup
+function setupAliases(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    getAll(function (error, mailboxes) {
+        if (error) return callback(error);
+
+        async.each(mailboxes, function (mailbox, iteratorDone) {
+            getAliases(mailbox.name, function (error, aliases) {
+                if (error) return iteratorDone(error);
+
+                if (aliases.length === 0) return iteratorDone();
+
+                pushAlias(mailbox.name, aliases, iteratorDone);
+            });
+        }, callback)
+    });
+}
+

src/mailer.js

@@ -41,6 +41,7 @@ var assert = require('assert'),
     ejs = require('ejs'),
     nodemailer = require('nodemailer'),
     path = require('path'),
+    platform = require('./platform.js'),
     safe = require('safetydance'),
     smtpTransport = require('nodemailer-smtp-transport'),
     users = require('./user.js'),
@@ -113,7 +114,7 @@ function getTxtRecords(callback) {
 function checkDns() {
     getTxtRecords(function (error, records) {
         if (error || !records) {
-            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.adminFqdn(), error, records);
+            debug('checkDns: DNS error or no records looking up TXT records for %s %s', config.fqdn(), error, records);
             gCheckDnsTimerId = setTimeout(checkDns, 60000);
             return;
         }
@@ -159,14 +160,14 @@ function sendMails(queue) {
 
         var transport = nodemailer.createTransport(smtpTransport({
             host: mailServerIp,
-            port: 2500, // this value comes from mail container
+            port: config.get('smtpPort'),
             auth: {
-                user: 'no-reply', // derive from adminEmail
-                pass: 'supersecret'
+                user: platform.mailConfig().username,
+                pass: platform.mailConfig().password
             }
         }));
 
-        debug('Processing mail queue of size %d (through %s:2500)', queue.length, mailServerIp);
+        debug('Processing mail queue of size %d (through %s:2525)', queue.length, mailServerIp);
 
         async.mapSeries(queue, function iterator(mailOptions, callback) {
             transport.sendMail(mailOptions, function (error) {
@@ -222,7 +223,7 @@ function mailUserEventToAdmins(user, event) {
         adminEmails = _.difference(adminEmails, [ user.email ]);
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s %s in Cloudron %s', user.username || user.email, event, config.fqdn()),
             text: render('user_event.ejs', { fqdn: config.fqdn(), user: user, event: event, format: 'text' }),
@@ -248,7 +249,7 @@ function sendInvite(user, invitor) {
     };
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: util.format('Welcome to Cloudron %s', config.fqdn()),
         text: render('welcome_user.ejs', templateData)
@@ -271,7 +272,7 @@ function userAdded(user, inviteSent) {
         var inviteLink = inviteSent ? null : config.adminOrigin() + '/api/v1/session/account/setup.html?reset_token=' + user.resetToken;
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s added in Cloudron %s', user.email, config.fqdn()),
             text: render('user_added.ejs', { fqdn: config.fqdn(), user: user, inviteLink: inviteLink, format: 'text' }),
@@ -306,7 +307,7 @@ function passwordReset(user) {
     var resetLink = config.adminOrigin() + '/api/v1/session/password/reset.html?reset_token=' + user.resetToken;
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: user.email,
         subject: 'Password Reset Request',
         text: render('password_reset.ejs', { fqdn: config.fqdn(), user: user, resetLink: resetLink, format: 'text' })
@@ -324,7 +325,7 @@ function appDied(app) {
         if (error) return console.log('Error getting admins', error);
 
         var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.concat('support@cloudron.io').join(', '),
             subject: util.format('App %s is down', app.location),
             text: render('app_down.ejs', { fqdn: config.fqdn(), title: app.manifest.title, appFqdn: config.appFqdn(app.location), format: 'text' })
@@ -342,7 +343,7 @@ function boxUpdateAvailable(newBoxVersion, changelog) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', config.fqdn()),
             text: render('box_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), newBoxVersion: newBoxVersion, changelog: changelog, format: 'text' })
@@ -360,7 +361,7 @@ function appUpdateAvailable(app, updateInfo) {
         if (error) return console.log('Error getting admins', error);
 
          var mailOptions = {
-            from: config.adminEmail(),
+            from: platform.mailConfig().from,
             to: adminEmails.join(', '),
             subject: util.format('%s has a new update available', app.fqdn),
             text: render('app_update_available.ejs', { fqdn: config.fqdn(), webadminUrl: config.adminOrigin(), app: app, updateInfo: updateInfo, format: 'text' })
@@ -374,7 +375,7 @@ function outOfDiskSpace(message) {
     assert.strictEqual(typeof message, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] Out of disk space alert', config.fqdn()),
         text: render('out_of_disk_space.ejs', { fqdn: config.fqdn(), message: message, format: 'text' })
@@ -388,7 +389,7 @@ function certificateRenewed(domain, message) {
     assert.strictEqual(typeof message, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] Certificate was %s renewed', domain, message ? 'not' : ''),
         text: render('certificate_renewed.ejs', { domain: domain, message: message, format: 'text' })
@@ -404,7 +405,7 @@ function unexpectedExit(program, context) {
     assert.strictEqual(typeof context, 'string');
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'admin@cloudron.io',
         subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
         text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
@@ -426,7 +427,7 @@ function sendFeedback(user, type, subject, description) {
         type === exports.FEEDBACK_TYPE_APP_ERROR);
 
     var mailOptions = {
-        from: config.adminEmail(),
+        from: platform.mailConfig().from,
         to: 'support@cloudron.io',
         subject: util.format('[%s] %s - %s', type, config.fqdn(), subject),
         text: render('feedback.ejs', { fqdn: config.fqdn(), type: type, user: user, subject: subject, description: description, format: 'text'})

src/middleware/index.js

@@ -4,7 +4,6 @@ exports = module.exports = {
     cookieParser: require('cookie-parser'),
     cors: require('./cors'),
     csrf: require('csurf'),
-    favicon: require('serve-favicon'),
     json: require('body-parser').json,
     morgan: require('morgan'),
     proxy: require('proxy-middleware'),

src/oauth2views/account_setup.ejs

@@ -8,7 +8,10 @@
 
 // very basic angular app
 var app = angular.module('Application', []);
-app.controller('Controller', [function () {}]);
+app.controller('Controller', ['$scope', function ($scope) {
+    $scope.username = '<%= (user && user.username) ? user.username : '' %>';
+    $scope.displayName = '<%= (user && user.displayName) ? user.displayName : '' %>';
+}]);
 
 </script>
 
@@ -28,6 +31,12 @@ app.controller('Controller', [function () {}]);
 
                 <center><p class="has-error"><%= error %></p></center>
 
+<% if (user && user.username) { %>
+                <div class="form-group"">
+                    <label class="control-label">Username</label>
+                    <input type="text" class="form-control" ng-model="username" name="username" readonly required>
+                </div>
+<% } else { %>
                 <div class="form-group" ng-class="{ 'has-error': (setupForm.username.$dirty && setupForm.username.$invalid) }">
                     <label class="control-label">Username</label>
                     <div class="control-label" ng-show="setupForm.username.$dirty && setupForm.username.$invalid">
@@ -37,6 +46,7 @@ app.controller('Controller', [function () {}]);
                     </div>
                     <input type="text" class="form-control" ng-model="username" name="username" ng-maxlength="512" ng-minlength="3" required autofocus>
                 </div>
+<% } %>
 
                 <div class="form-group">
                     <label class="control-label">Display Name</label>

src/oauthproxy.js

@@ -7,7 +7,7 @@ exports = module.exports = {
 
 var appdb = require('./appdb.js'),
     assert = require('assert'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     config = require('./config.js'),
     DatabaseError = require('./databaseerror.js'),
     debug = require('debug')('box:proxy'),
@@ -124,7 +124,7 @@ function authenticate(req, res, next) {
                 return res.send(500, 'Unknown app.');
             }
 
-            clientdb.getByAppIdAndType(result.id, clientdb.TYPE_PROXY, function (error, result) {
+            clients.getByAppIdAndType(result.id, clients.TYPE_PROXY, function (error, result) {
                 if (error) {
                     console.error('Unknown OAuth client.', error);
                     return res.send(500, 'Unknown OAuth client.');

src/paths.js

@@ -27,5 +27,7 @@ exports = module.exports = {
     UPDATE_CHECKER_FILE: path.join(config.baseDir(), 'data/box/updatechecker.json'),
 
     ACME_CHALLENGES_DIR: path.join(config.baseDir(), 'data/acme'),
-    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key')
+    ACME_ACCOUNT_KEY_FILE: path.join(config.baseDir(), 'data/box/acme/acme.key'),
+
+    INFRA_VERSION_FILE: path.join(config.baseDir(), 'data/INFRA_VERSION')
 };

src/platform.js

@@ -0,0 +1,245 @@
+'use strict';
+
+exports = module.exports = {
+    initialize: initialize,
+
+    mailConfig: mailConfig
+};
+
+var apps = require('./apps.js'),
+    assert = require('assert'),
+    async = require('async'),
+    config = require('./config.js'),
+    certificates = require('./certificates.js'),
+    debug = require('debug')('box:platform'),
+    fs = require('fs'),
+    hat = require('hat'),
+    infra = require('./infra_version.js'),
+    ini = require('ini'),
+    mailboxes = require('./mailboxes.js'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
+    shell = require('./shell.js'),
+    util = require('util');
+
+var gAddonVars = null;
+
+function initialize(callback) {
+    if (process.env.BOX_ENV === 'test' && !process.env.CREATE_INFRA) return callback();
+
+    debug('initializing addon infrastructure');
+
+    var existingInfra = { version: 'none' };
+    if (fs.existsSync(paths.INFRA_VERSION_FILE)) {
+        existingInfra = safe.JSON.parse(fs.readFileSync(paths.INFRA_VERSION_FILE, 'utf8'));
+        if (!existingInfra) existingInfra = { version: 'legacy' };
+    }
+
+    if (infra.version === existingInfra.version) {
+        debug('platform is uptodate at version %s', infra.version);
+        return loadAddonVars(callback);
+    }
+
+    debug('Updating infrastructure from %s to %s', existingInfra.version, infra.version);
+
+    async.series([
+        stopContainers,
+        createDockerNetwork,
+        startAddons,
+        removeOldImages,
+        existingInfra.version === 'none' ? apps.restoreInstalledApps : apps.configureInstalledApps,
+        loadAddonVars,
+        mailboxes.setupAliases,
+        fs.writeFile.bind(fs, paths.INFRA_VERSION_FILE, JSON.stringify(infra))
+    ], callback);
+}
+
+function removeOldImages(callback) {
+    debug('removing old addon images');
+
+    for (var imageName in infra.images) {
+        var image = infra.images[imageName];
+        debug('cleaning up images of %j', image);
+        var cmd = 'docker images "%s" | tail -n +2 | awk \'{ print $1 ":" $2 }\' | grep -v "%s" | xargs --no-run-if-empty docker rmi';
+        shell.execSync('removeOldImagesSync', util.format(cmd, image.repo, image.tag));
+    }
+
+    callback();
+}
+
+function stopContainers(callback) {
+    // TODO: be nice and stop addons cleanly (example, shutdown commands)
+    debug('stopping existing containers');
+    shell.execSync('stopContainersSync', 'docker ps -qa | xargs --no-run-if-empty docker rm -f');
+    callback();
+}
+
+function createDockerNetwork(callback) {
+    shell.execSync('createDockerNetwork', 'docker network create --subnet=172.18.0.0/16 cloudron || true', callback);
+}
+
+function startGraphite(callback) {
+    const tag = infra.images.graphite.tag;
+    const dataDir = paths.DATA_DIR;
+
+    const cmd = `docker run --restart=always -d --name="graphite" \
+                --net cloudron \
+                --net-alias graphite \
+                -m 75m \
+                --memory-swap 150m \
+                -p 127.0.0.1:2003:2003 \
+                -p 127.0.0.1:2004:2004 \
+                -p 127.0.0.1:8000:8000 \
+                -v "${dataDir}/graphite:/app/data" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startGraphite', cmd);
+
+    callback();
+}
+
+function startMysql(callback) {
+    const tag = infra.images.mysql.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 
+            'MYSQL_ROOT_PASSWORD=' + rootPassword +'\nMYSQL_ROOT_HOST=172.18.0.1', 'utf8')) {
+        return callback(new Error('Could not create mysql var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="mysql" \
+                --net cloudron \
+                --net-alias mysql \
+                -m 256m \
+                --memory-swap 512m \
+                -v "${dataDir}/mysql:/var/lib/mysql" \
+                -v "${dataDir}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startMysql', cmd);
+
+    callback();
+}
+
+function startPostgresql(callback) {
+    const tag = infra.images.postgresql.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'POSTGRESQL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create postgresql var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="postgresql" \
+                --net cloudron \
+                --net-alias postgresql \
+                -m 100m \
+                --memory-swap 200m \
+                -v "${dataDir}/postgresql:/var/lib/postgresql" \
+                -v "${dataDir}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startPostgresql', cmd);
+
+    callback();
+}
+
+function startMongodb(callback) {
+    const tag = infra.images.mongodb.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'MONGODB_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create mongodb var file:' + safe.error.message));
+    }
+
+    const cmd = `docker run --restart=always -d --name="mongodb" \
+                --net cloudron \
+                --net-alias mongodb \
+                -m 100m \
+                --memory-swap 200m \
+                -v "${dataDir}/mongodb:/var/lib/mongodb" \
+                -v "${dataDir}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
+                --read-only -v /tmp -v /run "${tag}"`;
+
+    shell.execSync('startMongodb', cmd);
+
+    callback();
+}
+
+function startMail(callback) {
+    // mail (note: 2525 is hardcoded in mail container and app use this port)
+    // MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
+    // MAIL_DOMAIN is the domain for which this server is relaying mails
+    // mail container uses /app/data for backed up data and /run for restart-able data
+
+    const tag = infra.images.mail.tag;
+    const dataDir = paths.DATA_DIR;
+    const rootPassword = hat(8 * 128);
+    const fqdn = config.fqdn();
+    const mailFqdn = config.adminFqdn();
+
+    if (!safe.fs.writeFileSync(paths.DATA_DIR + '/addons/mail_vars.sh',
+            'MAIL_ROOT_USERNAME=no-reply\nMAIL_ROOT_PASSWORD=' + rootPassword, 'utf8')) {
+        return callback(new Error('Could not create mail var file:' + safe.error.message));
+    }
+
+    certificates.getAdminCertificatePath(function (error, certFilePath, keyFilePath) {
+        if (error) return callback(error);
+
+        const cmd = `docker run --restart=always -d --name="mail" \
+                    --net cloudron \
+                    --net-alias mail \
+                    -m 75m \
+                    --memory-swap 150m \
+                    -e "MAIL_DOMAIN=${fqdn}" \
+                    -e "MAIL_SERVER_NAME=${mailFqdn}" \
+                    -v "${dataDir}/box/mail:/app/data" \
+                    -v "${dataDir}/mail:/run" \
+                    -v "${dataDir}/addons/mail_vars.sh:/etc/mail/mail_vars.sh:ro" \
+                    -v "${certFilePath}:/etc/tls_cert.pem:ro" \
+                    -v "${keyFilePath}:/etc/tls_key.pem:ro" \
+                    -p 587:2525 \
+                    -p 993:9993 \
+                    -p 4190:4190 \
+                    -p 25:2525 \
+                    --read-only -v /tmp ${tag}`;
+
+        shell.execSync('startMongodb', cmd);
+
+        callback();
+    });
+}
+
+function startAddons(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        startGraphite,
+        startMysql,
+        startPostgresql,
+        startMongodb,
+        startMail
+    ], callback);
+}
+
+function loadAddonVars(callback) {
+    gAddonVars = {
+        mail: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mail_vars.sh', 'utf8')),
+        postgresql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/postgresql_vars.sh', 'utf8')),
+        mysql: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mysql_vars.sh', 'utf8')),
+        mongodb: ini.parse(fs.readFileSync(paths.DATA_DIR + '/addons/mongodb_vars.sh', 'utf8'))
+    };
+    callback();
+}
+
+function mailConfig() {
+    if (!gAddonVars) return { username: 'no-reply', from: 'no-reply@' + config.fqdn(), password: 'doesnotwork' }; // for tests which don't run infra
+
+    return {
+        username: gAddonVars.mail.MAIL_ROOT_USERNAME,
+        from: '"Cloudron" <' + gAddonVars.mail.MAIL_ROOT_USERNAME + '@' + config.fqdn() + '>',
+        password: gAddonVars.mail.MAIL_ROOT_PASSWORD
+    };
+}

src/routes/apps.js

@@ -16,7 +16,9 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
-    exec: exec
+    exec: exec,
+
+    cloneApp: cloneApp
 };
 
 var apps = require('../apps.js'),
@@ -28,8 +30,7 @@ var apps = require('../apps.js'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     paths = require('../paths.js'),
     safe = require('safetydance'),
-    util = require('util'),
-    uuid = require('node-uuid');
+    util = require('util');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -90,82 +91,74 @@ function getAppIcon(req, res, next) {
     });
 }
 
-/*
- * Installs an app
- * @bodyparam {string} appStoreId The id of the app to be installed
- * @bodyparam {manifest} manifest The app manifest
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from environment variable name to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- * @bodyparam {icon} icon Base64 encoded image
- */
 function installApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId is required'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    // required
     if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
+
+    // optional
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
+
+    // falsy value in altDomain unsets it
     if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
-    // allow tests to provide an appId for testing
-    var appId = (process.env.BOX_ENV === 'test' && typeof data.appId === 'string') ? data.appId : uuid.v4();
+    debug('Installing app id:%s data:%j', data);
 
-    debug('Installing app id:%s storeid:%s loc:%s port:%j accessRestriction:%j memoryLimit:%s manifest:%j', appId, data.appStoreId, data.location, data.portBindings, data.accessRestriction, data.memoryLimit, data.manifest);
-
-    apps.install(appId, data.appStoreId, data.manifest, data.location, data.portBindings || null, data.accessRestriction, data.icon || null, data.cert || null, data.key || null, data.memoryLimit || 0, data.altDomain || null, auditSource(req), function (error) {
+    apps.install(data, auditSource(req), function (error, app) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BILLING_REQUIRED) return next(new HttpError(402, 'Billing required'));
-        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_CERTIFICATE) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.USER_REQUIRED) return next(new HttpError(400, 'accessRestriction must specify one user'));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(503, error.message));
         if (error) return next(new HttpError(500, error));
 
-        next(new HttpSuccess(202, { id: appId } ));
+        next(new HttpSuccess(202, app));
     });
 }
 
-/*
- * Configure an app
- * @bodyparam {string} password The user's password
- * @bodyparam {string} location The subdomain where the app is to be installed
- * @bodyparam {object} portBindings map from env to (public) host port. can be null.
-                       If a value in manifest.tcpPorts is missing in portBindings, the port/service is disabled
- */
 function configureApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
-    if (typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction is required'));
-    if (data.cert && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
-    if (data.key && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
+    if ('location' in data && typeof data.location !== 'string') return next(new HttpError(400, 'location must be string'));
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    if ('accessRestriction' in data && typeof data.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
+
+    // falsy values in cert and key unset the cert
+    if (data.key && typeof data.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
+    if (data.cert && typeof data.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (data.cert && !data.key) return next(new HttpError(400, 'key must be provided'));
     if (!data.cert && data.key) return next(new HttpError(400, 'cert must be provided'));
+
     if ('memoryLimit' in data && typeof data.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
     if (data.altDomain && typeof data.altDomain !== 'string') return next(new HttpError(400, 'altDomain must be a string'));
 
-    debug('Configuring app id:%s location:%s bindings:%j accessRestriction:%j memoryLimit:%s', req.params.id, data.location, data.portBindings, data.accessRestriction, data.memoryLimit);
+    debug('Configuring app id:%s data:%j', req.params.id, data);
 
-    apps.configure(req.params.id, data.location, data.portBindings || null, data.accessRestriction, data.cert || null, data.key || null, data.memoryLimit || 0, data.altDomain || null, auditSource(req), function (error) {
+    apps.configure(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === AppsError.PORT_RESERVED) return next(new HttpError(409, 'Port ' + error.message + ' is reserved.'));
         if (error && error.reason === AppsError.PORT_CONFLICT) return next(new HttpError(409, 'Port ' + error.message + ' is already in use.'));
@@ -180,20 +173,50 @@ function configureApp(req, res, next) {
 }
 
 function restoreApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.id, 'string');
 
+    var data = req.body;
+
     debug('Restore app id:%s', req.params.id);
 
-    apps.restore(req.params.id, auditSource(req), function (error) {
+    if (!('backupId' in req.body)) return next(new HttpError(400, 'backupId is required'));
+    if (data.backupId !== null && typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be string or null'));
+
+    apps.restore(req.params.id, data, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(202, { }));
     });
 }
 
+function cloneApp(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    var data = req.body;
+
+    debug('Clone app id:%s', req.params.id);
+
+    if (typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
+    if (typeof data.location !== 'string') return next(new HttpError(400, 'location is required'));
+    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+
+    apps.clone(req.params.id, data, auditSource(req), function (error, result) {
+        if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
+        if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
+        if (error && error.reason === AppsError.EXTERNAL_ERROR) return next(new HttpError(424, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, { id: result.id }));
+    });
+}
+
 function backupApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
@@ -209,10 +232,6 @@ function backupApp(req, res, next) {
     });
 }
 
-/*
- * Uninstalls an app
- * @bodyparam {string} id The id of the app to be uninstalled
- */
 function uninstallApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
@@ -260,15 +279,18 @@ function updateApp(req, res, next) {
 
     var data = req.body;
 
-    if (!data) return next(new HttpError(400, 'Cannot parse data field'));
-    if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest is required'));
-    if (('portBindings' in data) && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
+    // atleast one
+    if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
+    if ('appStoreId' in data && typeof data.appStoreId !== 'string') return next(new HttpError(400, 'appStoreId must be a string'));
+    if (!data.manifest && !data.appStoreId) return next(new HttpError(400, 'appStoreId or manifest is required'));
+
+    if ('portBindings' in data && typeof data.portBindings !== 'object') return next(new HttpError(400, 'portBindings must be an object'));
     if ('icon' in data && typeof data.icon !== 'string') return next(new HttpError(400, 'icon is not a string'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
     debug('Update app id:%s to manifest:%j with portBindings:%j', req.params.id, data.manifest, data.portBindings);
 
-    apps.update(req.params.id, data.force || false, data.manifest, data.portBindings || null, data.icon, auditSource(req), function (error) {
+    apps.update(req.params.id, req.body, auditSource(req), function (error) {
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(404, 'No such app'));
         if (error && error.reason === AppsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === AppsError.BAD_STATE) return next(new HttpError(409, error.message));
@@ -338,15 +360,37 @@ function getLogs(req, res, next) {
     });
 }
 
+function demuxStream(stream, stdin) {
+    var header = null;
+
+    stream.on('readable', function() {
+        header = header || stream.read(4);
+
+        while (header !== null) {
+            var length = header.readUInt32BE(0);
+            if (length === 0) {
+                header = null;
+                return stdin.end(); // EOF
+            }
+
+            var payload = stream.read(length);
+
+            if (payload === null) break;
+            stdin.write(payload);
+            header = stream.read(4);
+        }
+    });
+}
+
 function exec(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    debug('Execing into app id:%s', req.params.id);
+    debug('Execing into app id:%s and cmd:%s', req.params.id, req.query.cmd);
 
     var cmd = null;
     if (req.query.cmd) {
         cmd = safe.JSON.parse(req.query.cmd);
-        if (!util.isArray(cmd) && cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
+        if (!util.isArray(cmd) || cmd.length < 1) return next(new HttpError(400, 'cmd must be array with atleast size 1'));
     }
 
     var columns = req.query.columns ? parseInt(req.query.columns, 10) : null;
@@ -367,8 +411,16 @@ function exec(req, res, next) {
         req.clearTimeout();
         res.sendUpgradeHandshake();
 
+        // When tty is disabled, the duplexStream has 2 separate streams. When enabled, it has stdout/stderr merged.
         duplexStream.pipe(res.socket);
-        res.socket.pipe(duplexStream);
+
+        if (tty) {
+            res.socket.pipe(duplexStream); // in tty mode, the client always waits for server to exit
+        } else {
+            demuxStream(res.socket, duplexStream);
+            res.socket.on('error', function () { duplexStream.end(); });
+            res.socket.on('end', function () { duplexStream.end(); });
+        }
     });
 }
 

src/routes/backups.js

@@ -3,7 +3,7 @@
 exports = module.exports = {
     get: get,
     create: create,
-    download: download
+    createDownloadUrl: createDownloadUrl
 };
 
 var assert = require('assert'),
@@ -43,7 +43,7 @@ function create(req, res, next) {
     });
 }
 
-function download(req, res, next) {
+function createDownloadUrl(req, res, next) {
     assert.strictEqual(typeof req.params.backupId, 'string');
 
     backups.getRestoreUrl(req.params.backupId, function (error, result) {

src/routes/clients.js

@@ -4,16 +4,16 @@ exports = module.exports = {
     add: add,
     get: get,
     del: del,
-    getAllByUserId: getAllByUserId,
+    getAll: getAll,
+    addClientToken: addClientToken,
     getClientTokens: getClientTokens,
-    delClientTokens: delClientTokens
+    delClientTokens: delClientTokens,
+    delToken: delToken
 };
 
 var assert = require('assert'),
-    clientdb = require('../clientdb.js'),
     clients = require('../clients.js'),
     ClientsError = clients.ClientsError,
-    DatabaseError = require('../databaseerror.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     validUrl = require('valid-url');
@@ -27,8 +27,8 @@ function add(req, res, next) {
     if (typeof data.scope !== 'string' || !data.scope) return next(new HttpError(400, 'scope is required'));
     if (!validUrl.isWebUri(data.redirectURI)) return next(new HttpError(400, 'redirectURI must be a valid uri'));
 
-    clients.add(data.appId, clientdb.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
-        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, 'Invalid scope'));
+    clients.add(data.appId, clients.TYPE_EXTERNAL, data.redirectURI, data.scope, function (error, result) {
+        if (error && error.reason === ClientsError.INVALID_SCOPE) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(201, result));
     });
@@ -38,7 +38,7 @@ function get(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
     clients.get(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'No such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, result));
     });
@@ -47,26 +47,49 @@ function get(req, res, next) {
 function del(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
 
-    clients.del(req.params.clientId, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+    clients.get(req.params.clientId, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(204, result));
+
+        // we do not allow to use the REST API to delete addon clients
+        if (result.type !== clients.TYPE_EXTERNAL) return next(new HttpError(405, 'Deleting app addon clients is not allowed.'));
+
+        clients.del(req.params.clientId, function (error, result) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+            if (error && error.reason === ClientsError.NOT_ALLOWED) return next(new HttpError(405, error.message));
+            if (error) return next(new HttpError(500, error));
+            next(new HttpSuccess(204, result));
+        });
     });
 }
 
-function getAllByUserId(req, res, next) {
-    clients.getAllWithDetailsByUserId(req.user.id, function (error, result) {
-        if (error && error.reason !== DatabaseError.NOT_FOUND) return next(new HttpError(500, error));
+function getAll(req, res, next) {
+    clients.getAll(function (error, result) {
+        if (error && error.reason !== ClientsError.NOT_FOUND) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { clients: result }));
     });
 }
 
+function addClientToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    var expiresAt = req.query.expiresAt ? parseInt(req.query.expiresAt, 10) : Date.now() + 24 * 60 * 60 * 1000; // default 1 day;
+    if (isNaN(expiresAt) || expiresAt <= Date.now()) return next(new HttpError(400, 'expiresAt must be a timestamp in the future'));
+
+    clients.addClientTokenByUserId(req.params.clientId, req.user.id, expiresAt, function (error, result) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+        next(new HttpSuccess(201, { token: result }));
+    });
+}
+
 function getClientTokens(req, res, next) {
     assert.strictEqual(typeof req.params.clientId, 'string');
     assert.strictEqual(typeof req.user, 'object');
 
     clients.getClientTokensByUserId(req.params.clientId, req.user.id, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(200, { tokens: result }));
     });
@@ -77,8 +100,22 @@ function delClientTokens(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
     clients.delClientTokensByUserId(req.params.clientId, req.user.id, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(404, 'no such client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
         if (error) return next(new HttpError(500, error));
         next(new HttpSuccess(204));
     });
 }
+
+function delToken(req, res, next) {
+    assert.strictEqual(typeof req.params.clientId, 'string');
+    assert.strictEqual(typeof req.params.tokenId, 'string');
+    assert.strictEqual(typeof req.user, 'object');
+
+    clients.delToken(req.params.clientId, req.params.tokenId, function (error) {
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(404, error.message));
+        if (error && error.reason === ClientsError.INVALID_TOKEN) return next(new HttpError(404, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}

src/routes/cloudron.js

@@ -8,10 +8,12 @@ exports = module.exports = {
     getProgress: getProgress,
     getConfig: getConfig,
     update: update,
-    feedback: feedback
+    feedback: feedback,
+    checkForUpdates: checkForUpdates
 };
 
 var assert = require('assert'),
+    async = require('async'),
     cloudron = require('../cloudron.js'),
     CloudronError = cloudron.CloudronError,
     config = require('../config.js'),
@@ -20,7 +22,8 @@ var assert = require('assert'),
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     progress = require('../progress.js'),
     mailer = require('../mailer.js'),
-    superagent = require('superagent');
+    superagent = require('superagent'),
+    updateChecker = require('../updatechecker.js');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -55,9 +58,7 @@ function activate(req, res, next) {
 
     cloudron.activate(username, password, email, displayName, ip, auditSource(req), function (error, info) {
         if (error && error.reason === CloudronError.ALREADY_PROVISIONED) return next(new HttpError(409, 'Already setup'));
-        if (error && error.reason === CloudronError.BAD_USERNAME) return next(new HttpError(400, 'Bad username'));
-        if (error && error.reason === CloudronError.BAD_PASSWORD) return next(new HttpError(400, 'Bad password'));
-        if (error && error.reason === CloudronError.BAD_EMAIL) return next(new HttpError(400, 'Bad email'));
+        if (error && error.reason === CloudronError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         // only in caas case do we have to notify the api server about activation
@@ -131,6 +132,15 @@ function update(req, res, next) {
     });
 }
 
+function checkForUpdates(req, res, next) {
+    async.series([
+        updateChecker.checkAppUpdates,
+        updateChecker.checkBoxUpdates
+    ], function () {
+        next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
+    });
+}
+
 function feedback(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 

src/routes/developer.js

@@ -19,7 +19,7 @@ function auditSource(req) {
 }
 
 function enabled(req, res, next) {
-    developer.enabled(function (error, enabled) {
+    developer.isEnabled(function (error, enabled) {
         if (enabled) return next();
         next(new HttpError(412, 'Developer mode not enabled'));
     });

src/routes/groups.js

@@ -20,7 +20,7 @@ function create(req, res, next) {
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
 
     groups.create(req.body.name, function (error, group) {
-        if (error && error.reason === GroupError.BAD_NAME) return next(new HttpError(400, error.message));
+        if (error && error.reason === GroupError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === GroupError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
         if (error) return next(new HttpError(500, error));
 
@@ -45,7 +45,7 @@ function get(req, res, next) {
 }
 
 function list(req, res, next) {
-    groups.getAll(function (error, result) {
+    groups.getAllWithMembers(function (error, result) {
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200, { groups: result }));

src/routes/index.js

@@ -9,9 +9,10 @@ exports = module.exports = {
     eventlog: require('./eventlog.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
+    mailboxes: require('./mailboxes.js'),
     oauth2: require('./oauth2.js'),
     profile: require('./profile.js'),
-    settings: require('./settings.js'),
     sysadmin: require('./sysadmin.js'),
+    settings: require('./settings.js'),
     user: require('./user.js')
 };

src/routes/mailboxes.js

@@ -0,0 +1,91 @@
+'use strict';
+
+exports = module.exports = {
+    list: list,
+    get: get,
+    remove: remove,
+    create: create,
+    setAliases: setAliases,
+    getAliases: getAliases
+};
+
+var assert = require('assert'),
+    HttpError = require('connect-lastmile').HttpError,
+    HttpSuccess = require('connect-lastmile').HttpSuccess,
+    mailboxes = require('../mailboxes.js'),
+    MailboxError = mailboxes.MailboxError,
+    util = require('util');
+
+function create(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
+
+    mailboxes.add(req.body.name, function (error, mailbox) {
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(201, mailbox));
+    });
+}
+
+function get(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.get(req.params.mailboxId, function (error, result) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, result));
+    });
+}
+
+function list(req, res, next) {
+    mailboxes.getAll(function (error, result) {
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { mailboxes: result }));
+    });
+}
+
+function remove(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.del(req.params.mailboxId, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'Mailbox not found'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
+function setAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    if (!util.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
+
+    for (var i = 0; i < req.body.aliases.length; i++) {
+        if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
+    }
+
+    mailboxes.setAliases(req.params.mailboxId, req.body.aliases, function (error) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error && error.reason === MailboxError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === MailboxError.ALREADY_EXISTS) return next(new HttpError(409, 'One or more alias already exist'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
+function getAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.mailboxId, 'string');
+
+    mailboxes.getAliases(req.params.mailboxId, function (error, aliases) {
+        if (error && error.reason === MailboxError.NOT_FOUND) return next(new HttpError(404, 'No such mailbox'));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200, { aliases: aliases }));
+    });
+}

src/routes/oauth2.js

@@ -4,9 +4,9 @@ var appdb = require('../appdb'),
     apps = require('../apps'),
     assert = require('assert'),
     authcodedb = require('../authcodedb'),
-    clientdb = require('../clientdb'),
+    clients = require('../clients'),
+    ClientsError = clients.ClientsError,
     config = require('../config.js'),
-    constants = require('../constants.js'),
     DatabaseError = require('../databaseerror'),
     debug = require('debug')('box:routes/oauth2'),
     eventlog = require('../eventlog.js'),
@@ -21,7 +21,8 @@ var appdb = require('../appdb'),
     url = require('url'),
     user = require('../user.js'),
     UserError = user.UserError,
-    util = require('util');
+    util = require('util'),
+    _ = require('underscore');
 
 function auditSource(req, appId) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -41,7 +42,7 @@ gServer.serializeClient(function (client, callback) {
 });
 
 gServer.deserializeClient(function (id, callback) {
-    clientdb.get(id, callback);
+    clients.get(id, callback);
 });
 
 
@@ -76,7 +77,7 @@ gServer.grant(oauth2orize.grant.token({ scopeSeparator: ',' }, function (client,
     var token = tokendb.generateToken();
     var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-    tokendb.add(token, tokendb.PREFIX_USER + user.id, client.id, expires, client.scope, function (error) {
+    tokendb.add(token, user.id, client.id, expires, client.scope, function (error) {
         if (error) return callback(error);
 
         debug('grant token: new access token for client %s token %s', client.id, token);
@@ -106,7 +107,7 @@ gServer.exchange(oauth2orize.exchange.code(function (client, code, redirectURI,
             var token = tokendb.generateToken();
             var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-            tokendb.add(token, tokendb.PREFIX_USER + authCode.userId, authCode.clientId, expires, client.scope, function (error) {
+            tokendb.add(token, authCode.userId, authCode.clientId, expires, client.scope, function (error) {
                 if (error) return callback(error);
 
                 debug('exchange: new access token for client %s token %s', client.id, token);
@@ -201,13 +202,13 @@ function loginForm(req, res) {
         });
     }
 
-    clientdb.get(u.query.client_id, function (error, result) {
+    clients.get(u.query.client_id, function (error, result) {
         if (error) return sendError(req, res, 'Unknown OAuth client');
 
         switch (result.type) {
-            case clientdb.TYPE_ADMIN: return render(constants.ADMIN_NAME, '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_EXTERNAL: return render('External Application', '/api/v1/cloudron/avatar');
-            case clientdb.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
+            case clients.TYPE_BUILT_IN: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_EXTERNAL: return render(result.appId, '/api/v1/cloudron/avatar');
+            case clients.TYPE_SIMPLE_AUTH: return sendError(req, res, 'Unknown OAuth client');
             default: break;
         }
 
@@ -306,16 +307,20 @@ function accountSetup(req, res, next) {
     user.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return sendError(req, res, 'Invalid Reset Token');
 
-        userObject.username = req.body.username;
-        userObject.displayName = req.body.displayName;
-
-        user.update(userObject.id, userObject.username, userObject.email, userObject.displayName, auditSource(req), function (error) {
+        var data = _.pick(req.body, 'username', 'displayName');
+        user.update(userObject.id, data, auditSource(req), function (error) {
             if (error && error.reason === UserError.ALREADY_EXISTS) return renderAccountSetupSite(res, req, userObject, 'Username already exists');
+            if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+            if (error && error.reason === UserError.NOT_FOUND) return renderAccountSetupSite(res, req, userObject, 'No such user');
             if (error) return next(new HttpError(500, error));
 
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
+
             // setPassword clears the resetToken
             user.setPassword(userObject.id, req.body.password, function (error, result) {
-                if (error && error.reason === UserError.BAD_PASSWORD) return renderAccountSetupSite(res, req, userObject, 'Password invalid');
+                if (error && error.reason === UserError.BAD_FIELD) return renderAccountSetupSite(res, req, userObject, error.message);
+
                 if (error) return next(new HttpError(500, error));
 
                 res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -357,7 +362,7 @@ function passwordReset(req, res, next) {
 
         // setPassword clears the resetToken
         user.setPassword(userObject.id, req.body.password, function (error, result) {
-            if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(406, 'Password does not meet the requirements'));
+            if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(406, error.message));
             if (error) return next(new HttpError(500, error));
 
             res.redirect(util.format('%s?accessToken=%s&expiresAt=%s', config.adminOrigin(), result.token, result.expiresAt));
@@ -399,8 +404,8 @@ var authorization = [
     gServer.authorization({}, function (clientId, redirectURI, callback) {
         debug('authorization: client %s with callback to %s.', clientId, redirectURI);
 
-        clientdb.get(clientId, function (error, client) {
-            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null, false);
+        clients.get(clientId, function (error, client) {
+            if (error && error.reason === ClientsError.NOT_FOUND) return callback(null, false);
             if (error) return callback(error);
 
             // ignore the origin passed into form the client, but use the one from the clientdb
@@ -414,12 +419,12 @@ var authorization = [
         // Handle our different types of oauth clients
         var type = req.oauth2.client.type;
 
-        if (type === clientdb.TYPE_ADMIN) {
-            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, 'admin'), { userId: req.oauth2.user.id });
+        if (type === clients.TYPE_EXTERNAL || type === clients.TYPE_BUILT_IN) {
+            eventlog.add(eventlog.ACTION_USER_LOGIN, auditSource(req, req.oauth2.client.appId), { userId: req.oauth2.user.id });
             return next();
+        } else if (type === clients.TYPE_SIMPLE_AUTH) {
+            return sendError(req, res, 'Unknown OAuth client.');
         }
-        if (type === clientdb.TYPE_EXTERNAL) return next();
-        if (type === clientdb.TYPE_SIMPLE_AUTH) return sendError(req, res, 'Unknown OAuth client.');
 
         appdb.get(req.oauth2.client.appId, function (error, appObject) {
             if (error) return sendErrorPageOrRedirect(req, res, 'Invalid request. Unknown app for this client_id.');
@@ -451,6 +456,31 @@ var token = [
     gServer.errorHandler()
 ];
 
+// tests if all requestedScopes are attached to the request
+function validateRequestedScopes(req, requestedScopes) {
+    assert.strictEqual(typeof req, 'object');
+    assert(Array.isArray(requestedScopes));
+
+    if (!req.authInfo || !req.authInfo.scope) return new Error('No scope found');
+
+    var scopes = req.authInfo.scope.split(',');
+
+    // check for roles separately
+    if (requestedScopes.indexOf(clients.SCOPE_ROLE_SDK) !== -1 && scopes.indexOf(clients.SCOPE_ROLE_SDK) === -1) {
+        return new Error('Missing required scope role "' + clients.SCOPE_ROLE_SDK + '"');
+    }
+
+    if (scopes.indexOf('*') !== -1) return null;
+
+    for (var i = 0; i < requestedScopes.length; ++i) {
+        if (scopes.indexOf(requestedScopes[i]) === -1) {
+            debug('scope: missing scope "%s".', requestedScopes[i]);
+            return new Error('Missing required scope "' + requestedScopes[i] + '"');
+        }
+    }
+
+    return null;
+}
 
 //  The scope middleware provides an auth middleware for routes.
 //
@@ -470,17 +500,8 @@ function scope(requestedScope) {
     return [
         passport.authenticate(['bearer'], { session: false }),
         function (req, res, next) {
-            if (!req.authInfo || !req.authInfo.scope) return next(new HttpError(401, 'No scope found'));
-            if (req.authInfo.scope === '*') return next();
-
-            var scopes = req.authInfo.scope.split(',');
-
-            for (var i = 0; i < requestedScopes.length; ++i) {
-                if (scopes.indexOf(requestedScopes[i]) === -1) {
-                    debug('scope: missing scope "%s".', requestedScopes[i]);
-                    return next(new HttpError(401, 'Missing required scope "' + requestedScopes[i] + '"'));
-                }
-            }
+            var error = validateRequestedScopes(req, requestedScopes);
+            if (error) return next(new HttpError(401, error.message));
 
             next();
         }
@@ -511,6 +532,7 @@ exports = module.exports = {
     accountSetup: accountSetup,
     authorization: authorization,
     token: token,
+    validateRequestedScopes: validateRequestedScopes,
     scope: scope,
     csrf: csrf
 };

src/routes/profile.js

@@ -12,8 +12,8 @@ var assert = require('assert'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     user = require('../user.js'),
-    tokendb = require('../tokendb.js'),
-    UserError = user.UserError;
+    UserError = user.UserError,
+    _ = require('underscore');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -23,26 +23,18 @@ function auditSource(req) {
 function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    var result = {};
-    result.id = req.user.id;
-    result.tokenType = req.user.tokenType;
+    groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
+        if (error) return next(new HttpError(500, error));
 
-    if (req.user.tokenType === tokendb.TYPE_USER || req.user.tokenType === tokendb.TYPE_DEV) {
-        result.username = req.user.username;
-        result.email = req.user.email;
-        result.displayName = req.user.displayName;
-        result.showTutorial = req.user.showTutorial;
-
-        groups.isMember(groups.ADMIN_GROUP_ID, req.user.id, function (error, isAdmin) {
-            if (error) return next(new HttpError(500, error));
-
-            result.admin = isAdmin;
-
-            next(new HttpSuccess(200, result));
-        });
-    } else {
-        next(new HttpSuccess(200, result));
-    }
+        next(new HttpSuccess(200, {
+            id: req.user.id,
+            username: req.user.username,
+            email: req.user.email,
+            admin: isAdmin,
+            displayName: req.user.displayName,
+            showTutorial: req.user.showTutorial
+        }));
+    });
 }
 
 function update(req, res, next) {
@@ -52,12 +44,11 @@ function update(req, res, next) {
     if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
 
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+    var data = _.pick(req.body, 'email', 'displayName');
 
-    user.update(req.user.id, req.user.username, req.body.email || req.user.email, req.body.displayName || req.user.displayName, auditSource(req), function (error) {
-        if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+    user.update(req.user.id, data, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
         if (error) return next(new HttpError(500, error));
 
@@ -69,13 +60,10 @@ function changePassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.user, 'object');
 
-    if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires the users old password.'));
-    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'API call requires the users new password.'));
-
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
+    if (typeof req.body.newPassword !== 'string') return next(new HttpError(400, 'newPassword must be a string'));
 
     user.setPassword(req.user.id, req.body.newPassword, function (error) {
-        if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(403, 'Wrong password'));
         if (error) return next(new HttpError(500, error));
 

src/routes/settings.js

@@ -17,6 +17,7 @@ exports = module.exports = {
     setBackupConfig: setBackupConfig,
 
     getTimeZone: getTimeZone,
+    setTimeZone: setTimeZone,
 
     setCertificate: setCertificate,
     setAdminCertificate: setAdminCertificate
@@ -45,7 +46,7 @@ function setAutoupdatePattern(req, res, next) {
     if (typeof req.body.pattern !== 'string') return next(new HttpError(400, 'pattern is required'));
 
     settings.setAutoupdatePattern(req.body.pattern, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid pattern'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
 
         next(new HttpSuccess(200));
@@ -58,8 +59,9 @@ function setCloudronName(req, res, next) {
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name is required'));
 
     settings.setCloudronName(req.body.name, function (error) {
-        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, 'Invalid name'));
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200));
     });
 }
@@ -67,6 +69,7 @@ function setCloudronName(req, res, next) {
 function getCloudronName(req, res, next) {
     settings.getCloudronName(function (error, name) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, { name: name }));
     });
 }
@@ -74,10 +77,24 @@ function getCloudronName(req, res, next) {
 function getTimeZone(req, res, next) {
     settings.getTimeZone(function (error, tz) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(200, { timeZone: tz }));
     });
 }
 
+function setTimeZone(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.timeZone !== 'string') return next(new HttpError(400, 'timeZone is required'));
+
+    settings.setTimeZone(req.body.timeZone, function (error) {
+        if (error && error.reason === SettingsError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error) return next(new HttpError(500, error));
+
+        next(new HttpSuccess(200));
+    });
+}
+
 function setCloudronAvatar(req, res, next) {
     assert.strictEqual(typeof req.files, 'object');
 
@@ -86,6 +103,7 @@ function setCloudronAvatar(req, res, next) {
 
     settings.setCloudronAvatar(avatar, function (error) {
         if (error) return next(new HttpError(500, error));
+
         next(new HttpSuccess(202, {}));
     });
 }

src/routes/test/apps-test.js

@@ -1,7 +1,7 @@
 'use strict';
 
-/* jslint node:true */
 /* global it:false */
+/* global xit:false */
 /* global describe:false */
 /* global before:false */
 /* global after:false */
@@ -12,7 +12,7 @@ var appdb = require('../../appdb.js'),
     path = require('path'),
     async = require('async'),
     child_process = require('child_process'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     config = require('../../config.js'),
     constants = require('../../constants.js'),
     database = require('../../database.js'),
@@ -23,18 +23,18 @@ var appdb = require('../../appdb.js'),
     http = require('http'),
     https = require('https'),
     js2xml = require('js2xmlparser'),
+    ldap = require('../../ldap.js'),
     net = require('net'),
     nock = require('nock'),
     paths = require('../../paths.js'),
-    redis = require('redis'),
-    superagent = require('superagent'),
     safe = require('safetydance'),
     server = require('../../server.js'),
     settings = require('../../settings.js'),
+    simpleauth = require('../../simpleauth.js'),
+    superagent = require('superagent'),
     taskmanager = require('../../taskmanager.js'),
     tokendb = require('../../tokendb.js'),
     url = require('url'),
-    util = require('util'),
     uuid = require('node-uuid'),
     _ = require('underscore');
 
@@ -42,9 +42,9 @@ var SERVER_URL = 'http://localhost:' + config.get('port');
 
 // Test image information
 var TEST_IMAGE_REPO = 'cloudron/test';
-var TEST_IMAGE_TAG = '11.0.0';
+var TEST_IMAGE_TAG = '16.0.0';
 var TEST_IMAGE = TEST_IMAGE_REPO + ':' + TEST_IMAGE_TAG;
-var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
+// var TEST_IMAGE_ID = child_process.execSync('docker inspect --format={{.Id}} ' + TEST_IMAGE).toString('utf8').trim();
 
 var APP_STORE_ID = 'test', APP_ID;
 var APP_LOCATION = 'appslocation';
@@ -59,7 +59,7 @@ APP_MANIFEST_1.dockerImage = TEST_IMAGE;
 APP_MANIFEST_1.singleUser = true;
 
 var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='admin@me.com';
-var USER_1_ID = null, USERNAME_1 = 'user', PASSWORD_1 = 'Foobar?1338', EMAIL_1 ='user@me.com';
+var USER_1_ID = null, USERNAME_1 = 'user', EMAIL_1 ='user@me.com';
 var token = null; // authentication token
 var token_1 = null;
 
@@ -104,6 +104,34 @@ function startDockerProxy(interceptor, callback) {
     }).listen(5687, callback);
 }
 
+function checkAddons(appEntry, done) {
+    async.retry({ times: 15, interval: 3000 }, function (callback) {
+        // this was previously written with superagent but it was getting sporadic EPIPE
+        var req = http.get({ hostname: 'localhost', port: appEntry.httpPort, path: '/check_addons?username=' + USERNAME + '&password=' + PASSWORD });
+        req.on('error', callback);
+        req.on('response', function (res) {
+            if (res.statusCode !== 200) return callback('app returned non-200 status : ' + res.statusCode);
+
+            var d = '';
+            res.on('data', function (chunk) { d += chunk.toString('utf8'); });
+            res.on('end', function () {
+                var body = JSON.parse(d);
+
+                delete body.recvmail; // unclear why dovecot mail delivery won't work
+                delete body.stdenv; // cannot access APP_ORIGIN
+
+                for (var key in body) {
+                    if (body[key] !== 'OK') return callback('Not done yet: ' + JSON.stringify(body));
+                }
+
+                callback();
+            });
+        });
+
+        req.end();
+    }, done);
+}
+
 describe('Apps', function () {
     this.timeout(50000);
 
@@ -112,44 +140,21 @@ describe('Apps', function () {
     var imageCreated = false;
 
     before(function (done) {
-        console.log('Starting addons, this can take 10 seconds');
-
-        dockerProxy = startDockerProxy(function interceptor(req, res) {
-            if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
-                imageCreated = true;
-                res.writeHead(200);
-                res.end();
-                return true;
-            } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
-                imageDeleted = true;
-                res.writeHead(200);
-                res.end();
-                return true;
-            }
-            return false;
-        }, done);
-    });
-
-    after(function (done) {
-        // child_process.exec('docker rm -f mysql; docker rm -f postgresql; docker rm -f mongodb; docker rm -f mail', function (error) {
-        child_process.exec('docker ps | awk \'{print $1}\' | xargs docker rm -f', function () {
-            dockerProxy.close(done);
-        });
-    });
-
-
-    /*
-        Individual sub category setup and cleanup
-    */
-    function setup(done) {
         config._reset();
 
+        process.env.CREATE_INFRA = 1;
+
+        safe.fs.unlinkSync(paths.INFRA_VERSION_FILE);
+        child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+
         async.series([
             // first clear, then start server. otherwise, taskmanager spins up tasks for obsolete appIds
             database.initialize,
             database._clear,
 
             server.start.bind(server),
+            ldap.start,
+            simpleauth.start,
 
             function (callback) {
                 var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
@@ -188,58 +193,104 @@ describe('Apps', function () {
                 token_1 = tokendb.generateToken();
 
                 // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-                tokendb.add(token_1, tokendb.PREFIX_USER + USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+                tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+            },
+
+            function (callback) {
+                dockerProxy = startDockerProxy(function interceptor(req, res) {
+                    if (req.method === 'POST' && req.url === '/images/create?fromImage=' + encodeURIComponent(TEST_IMAGE_REPO) + '&tag=' + TEST_IMAGE_TAG) {
+                        imageCreated = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    } else if (req.method === 'DELETE' && req.url === '/images/' + TEST_IMAGE + '?force=false&noprune=false') {
+                        imageDeleted = true;
+                        res.writeHead(200);
+                        res.end();
+                        return true;
+                    }
+                    return false;
+                }, callback);
             },
 
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
             settings.setTlsConfig.bind(null, { provider: 'caas' }),
             settings.setBackupConfig.bind(null, { provider: 'caas', token: 'BACKUP_TOKEN', bucket: 'Bucket', prefix: 'Prefix' })
-        ], done);
-    }
+        ], function (error) {
+            if (error) return done(error);
+
+            console.log('This test can take ~40 seconds to start as it waits for infra to be ready');
+            setTimeout(done, 40000);
+        });
+    });
+
+    after(function (done) {
+        delete process.env.CREATE_INFRA;
+
+        // child_process.execSync('docker ps -qa | xargs --no-run-if-empty docker rm -f');
+        dockerProxy.close(function () { });
 
-    function cleanup(done) {
         // db is not cleaned up here since it's too late to call it after server.stop. if called before server.stop taskmanager apptasks are unhappy :/
         async.series([
             taskmanager.stopPendingTasks,
             taskmanager.waitForPendingTasks,
             server.stop,
+            ldap.stop,
+            simpleauth.stop,
             config._reset,
         ], done);
-    }
+    });
 
     describe('App API', function () {
-        this.timeout(50000);
-
-        before(setup);
-
         after(function (done) {
-            APP_ID = null;
-            cleanup(done);
+            appdb._clear(done); // TODO: test proper uninstall (requires mock for aws)
         });
 
         it('app install fails - missing manifest', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD })
+                   .send({ password: PASSWORD })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('manifest is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                 done();
             });
         });
 
-        it('app install fails - missing appId', function (done) {
+        it('app install fails - null manifest', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ manifest: APP_MANIFEST, password: PASSWORD })
+                   .send({ manifest: null, password: PASSWORD })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
-                expect(res.body.message).to.eql('appStoreId is required');
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
                 done();
             });
         });
 
-        it('app install fails - invalid json', function (done) {
+        it('app install fails - bad manifest format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: 'epic', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('manifest must be an object');
+                done();
+            });
+        });
+
+        it('app install fails - empty appStoreId format', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ manifest: null, appStoreId: '', password: PASSWORD })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(res.body.message).to.eql('appStoreId or manifest is required');
+                done();
+            });
+        });
+
+       it('app install fails - invalid json', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
                    .send('garbage')
@@ -252,7 +303,7 @@ describe('Apps', function () {
         it('app install fails - invalid location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: '!awesome', accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('Hostname can only contain alphanumerics and hyphen');
@@ -263,7 +314,7 @@ describe('Apps', function () {
         it('app install fails - invalid location type', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: 42, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('location is required');
@@ -274,7 +325,7 @@ describe('Apps', function () {
         it('app install fails - reserved admin location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.ADMIN_LOCATION, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql(constants.ADMIN_LOCATION + ' is reserved');
@@ -285,7 +336,7 @@ describe('Apps', function () {
         it('app install fails - reserved api location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: constants.API_LOCATION, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql(constants.API_LOCATION + ' is reserved');
@@ -296,7 +347,7 @@ describe('Apps', function () {
         it('app install fails - portBindings must be object', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: 23, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('portBindings must be an object');
@@ -307,7 +358,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction is required', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {} })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction is required');
@@ -318,7 +369,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction type is wrong', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: '' })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction is required');
@@ -329,7 +380,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction no users not allowed', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -340,7 +391,7 @@ describe('Apps', function () {
         it('app install fails - accessRestriction too many users not allowed', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
+                   .send({ manifest: APP_MANIFEST_1, password: PASSWORD, location: APP_LOCATION, portBindings: {}, accessRestriction: { users: [ 'one', 'two' ] } })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 expect(res.body.message).to.eql('accessRestriction must specify one user');
@@ -351,50 +402,64 @@ describe('Apps', function () {
         it('app install fails for non admin', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token_1 })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(403);
                 done();
             });
         });
 
-        it('app install fails due to purchase failure', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+        it('app install fails because manifest download fails', function (done) {
+            var fake = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(404, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .end(function (err, res) {
+                expect(res.statusCode).to.equal(400);
+                expect(fake.isDone()).to.be.ok();
+                done();
+            });
+        });
+
+        it('app install fails due to purchase failure', function (done) {
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(402, {});
+
+            superagent.post(SERVER_URL + '/api/v1/apps/install')
+                   .query({ access_token: token })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(402);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install succeeds with purchase', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: { users: [ 'someuser' ], groups: [] } })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 expect(res.body.id).to.be.a('string');
                 APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install fails because of conflicting location', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                   .send({ manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(409);
-                expect(fake.isDone()).to.be.ok();
                 done();
             });
         });
@@ -492,23 +557,23 @@ describe('Apps', function () {
         });
 
         it('app install succeeds already purchased', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                    .query({ access_token: token })
-                   .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
+                   .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION_2, portBindings: null, accessRestriction: null })
                    .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 expect(res.body.id).to.be.a('string');
                 APP_ID = res.body.id;
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 done();
             });
         });
 
         it('app install succeeds without password but developer token', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
-
             settings.setDeveloperMode(true, function (error) {
                 expect(error).to.be(null);
 
@@ -517,7 +582,7 @@ describe('Apps', function () {
                        .end(function (error, result) {
                     expect(error).to.not.be.ok();
                     expect(result.statusCode).to.equal(200);
-                    expect(result.body.expiresAt).to.be.a('number');
+                    expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                     expect(result.body.token).to.be.a('string');
 
                     // overwrite non dev token
@@ -525,11 +590,10 @@ describe('Apps', function () {
 
                     superagent.post(SERVER_URL + '/api/v1/apps/install')
                            .query({ access_token: token })
-                           .send({ appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
+                           .send({ manifest: APP_MANIFEST, location: APP_LOCATION+APP_LOCATION, portBindings: null, accessRestriction: null })
                            .end(function (err, res) {
                         expect(res.statusCode).to.equal(202);
                         expect(res.body.id).to.be.a('string');
-                        expect(fake.isDone()).to.be.ok();
                         APP_ID = res.body.id;
                         done();
                     });
@@ -560,8 +624,6 @@ describe('Apps', function () {
             imageCreated = false;
 
             async.series([
-                setup,
-
                 function (callback) {
                     apiHockInstance
                         .get('/api/v1/apps/' + APP_STORE_ID + '/versions/' + APP_MANIFEST.version + '/icon')
@@ -590,7 +652,6 @@ describe('Apps', function () {
             APP_ID = null;
 
             async.series([
-                cleanup,
                 apiHockServer.close.bind(apiHockServer),
                 awsHockServer.close.bind(awsHockServer)
             ], done);
@@ -599,7 +660,8 @@ describe('Apps', function () {
         var appResult = null /* the json response */, appEntry = null /* entry from database */;
 
         it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(200, {});
 
             var count = 0;
             function checkInstallStatus() {
@@ -616,12 +678,13 @@ describe('Apps', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: null, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
                 expect(res.body.id).to.be.a('string');
-                expect(res.body.id).to.be.eql(APP_ID);
+                APP_ID = res.body.id;
                 checkInstallStatus();
             });
         });
@@ -651,14 +714,7 @@ describe('Apps', function () {
                 expect(data.Config.Env).to.contain('APP_ORIGIN=https://' + config.appFqdn(APP_LOCATION));
                 expect(data.Config.Env).to.contain('APP_DOMAIN=' + config.appFqdn(APP_LOCATION));
                 expect(data.Config.Hostname).to.be(APP_LOCATION);
-                clientdb.getByAppIdAndType(appResult.id, clientdb.TYPE_OAUTH, function (error, client) {
-                    expect(error).to.not.be.ok();
-                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
-                    expect(client.clientSecret.length).to.be(64); // 32 hex chars (256 bits)
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
-                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
-                    done();
-                });
+                done();
             });
         });
 
@@ -704,6 +760,48 @@ describe('Apps', function () {
             });
         });
 
+        it('installation - app responnds to http request', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort).end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                expect(res.body.status).to.be('OK');
+                done();
+            });
+        });
+
+        it('installation - oauth addon config', function (done) {
+            var appContainer = docker.getContainer(appEntry.containerId);
+            appContainer.inspect(function (error, data) {
+                expect(error).to.not.be.ok();
+
+                clients.getByAppIdAndType(APP_ID, clients.TYPE_OAUTH, function (error, client) {
+                    expect(error).to.not.be.ok();
+                    expect(client.id.length).to.be(40); // cid- + 32 hex chars (128 bits) + 4 hyphens
+                    expect(client.clientSecret.length).to.be(256); // 32 hex chars (8 * 256 bits)
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_ID=' + client.id);
+                    expect(data.Config.Env).to.contain('OAUTH_CLIENT_SECRET=' + client.clientSecret);
+                    done();
+                });
+            });
+        });
+
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
+                done();
+            });
+        });
+
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 1');
+            checkAddons(appEntry, done);
+        });
+
         var redisIp, exportedRedisPort;
 
         it('installation - redis addon created', function (done) {
@@ -711,7 +809,7 @@ describe('Apps', function () {
                 expect(error).to.not.be.ok();
                 expect(data).to.be.ok();
 
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
+                redisIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
                 expect(redisIp).to.be.ok();
 
                 exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
@@ -721,129 +819,6 @@ describe('Apps', function () {
             });
         });
 
-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('installation - mysql addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var mysqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('MYSQL_URL=') === 0) mysqlUrl = env.split('=')[1]; });
-                expect(mysqlUrl).to.be.ok();
-
-                var urlp = url.parse(mysqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('MYSQL_PORT=3306');
-                expect(data.Config.Env).to.contain('MYSQL_HOST=mysql');
-                expect(data.Config.Env).to.contain('MYSQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('MYSQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('MYSQL_DATABASE=' + dbname);
-
-                var cmd = util.format('mysql -h %s -u%s -p%s --database=%s -e "CREATE TABLE IF NOT EXISTS foo (id INT);"',
-                    'mysql', username, password, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    // expect(stderr.length).to.be(0); // "Warning: Using a password on the command line interface can be insecure."
-                    done();
-                });
-            });
-        });
-
-        it('installation - postgresql addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var postgresqlUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('POSTGRESQL_URL=') === 0) postgresqlUrl = env.split('=')[1]; });
-                expect(postgresqlUrl).to.be.ok();
-
-                var urlp = url.parse(postgresqlUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('POSTGRESQL_PORT=5432');
-                expect(data.Config.Env).to.contain('POSTGRESQL_HOST=postgresql');
-                expect(data.Config.Env).to.contain('POSTGRESQL_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('POSTGRESQL_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('POSTGRESQL_DATABASE=' + dbname);
-
-                var cmd = util.format('bash -c "PGPASSWORD=%s psql -q -h %s -U%s --dbname=%s -e \'CREATE TABLE IF NOT EXISTS foo (id INT);\'"',
-                    password, 'postgresql', username, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error, stdout, stderr) {
-                    expect(!error).to.be.ok();
-                    expect(stdout.length).to.be(0);
-                    expect(stderr.length).to.be(0);
-                    done();
-                });
-            });
-        });
-
-        it('installation - mongodb addon config', function (done) {
-            var appContainer = docker.getContainer(appEntry.containerId);
-            appContainer.inspect(function (error, data) {
-                var mongodbUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('MONGODB_URL=') === 0) mongodbUrl = env.split('=')[1]; });
-                expect(mongodbUrl).to.be.ok();
-
-                var urlp = url.parse(mongodbUrl);
-                var username = urlp.auth.split(':')[0];
-                var password = urlp.auth.split(':')[1];
-                var dbname = urlp.path.substr(1);
-
-                expect(data.Config.Env).to.contain('MONGODB_PORT=27017');
-                expect(data.Config.Env).to.contain('MONGODB_HOST=mongodb');
-                expect(data.Config.Env).to.contain('MONGODB_USERNAME=' + username);
-                expect(data.Config.Env).to.contain('MONGODB_PASSWORD=' + password);
-                expect(data.Config.Env).to.contain('MONGODB_DATABASE=' + dbname);
-
-                var cmd = util.format('mongo --quiet -u %s -p %s %s:%s/%s --eval "db.collection.insert({ item: 34 })"',
-                    username, password, 'mongodb', 27017, dbname);
-
-                child_process.exec('docker exec ' + appContainer.id + ' ' + cmd, { timeout: 5000 }, function (error) {
-                    expect(!error).to.be.ok();
-                    done();
-                });
-            });
-        });
-
-        it('installation - scheduler', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (retryCallback) {
-                if (fs.existsSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env')) return retryCallback();
-
-                retryCallback(new Error('not run yet'));
-            }, done);
-        });
-
         xit('logs - stdout and stderr', function (done) {
             superagent.get(SERVER_URL + '/api/v1/apps/' + APP_ID + '/logs')
                 .query({ access_token: token })
@@ -963,6 +938,12 @@ describe('Apps', function () {
             checkStartState();
         });
 
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 2');
+            checkAddons(appEntry, done);
+        });
+
         it('can uninstall app', function (done) {
             var count = 0;
             function checkUninstallStatus() {
@@ -1046,8 +1027,6 @@ describe('Apps', function () {
             APP_ID = uuid.v4();
 
             async.series([
-                setup,
-
                 function (callback) {
                     config.set('fqdn', 'test.foobar.com');
                     callback();
@@ -1084,7 +1063,6 @@ describe('Apps', function () {
         after(function (done) {
             APP_ID = null;
             async.series([
-                cleanup,
                 apiHockServer.close.bind(apiHockServer),
                 awsHockServer.close.bind(awsHockServer)
             ], done);
@@ -1093,7 +1071,8 @@ describe('Apps', function () {
         var appResult = null, appEntry = null;
 
         it('can install test app', function (done) {
-            var fake = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
+            var fake1 = nock(config.apiServerOrigin()).get('/api/v1/apps/test').reply(200, { manifest: APP_MANIFEST });
+            var fake2 = nock(config.apiServerOrigin()).post('/api/v1/apps/test/purchase?token=APPSTORE_TOKEN').reply(201, {});
 
             var count = 0;
             function checkInstallStatus() {
@@ -1110,11 +1089,12 @@ describe('Apps', function () {
 
             superagent.post(SERVER_URL + '/api/v1/apps/install')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, appStoreId: APP_STORE_ID, manifest: APP_MANIFEST, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
+                  .send({ appStoreId: APP_STORE_ID, password: PASSWORD, location: APP_LOCATION, portBindings: { ECHO_SERVER_PORT: 7171 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
-                expect(fake.isDone()).to.be.ok();
-                expect(res.body.id).to.equal(APP_ID);
+                expect(fake1.isDone()).to.be.ok();
+                expect(fake2.isDone()).to.be.ok();
+                APP_ID = res.body.id;
                 checkInstallStatus();
             });
         });
@@ -1202,6 +1182,24 @@ describe('Apps', function () {
             });
         });
 
+
+        it('installation - app can populate addons', function (done) {
+            superagent.get('http://localhost:' + appEntry.httpPort + '/populate_addons').end(function (err, res) {
+                expect(!err).to.be.ok();
+                expect(res.statusCode).to.equal(200);
+                for (var key in res.body) {
+                    expect(res.body[key]).to.be('OK');
+                }
+                done();
+            });
+        });
+
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 3');
+            checkAddons(appEntry, done);
+        });
+
         var redisIp, exportedRedisPort;
 
         it('installation - redis addon created', function (done) {
@@ -1209,7 +1207,7 @@ describe('Apps', function () {
                 expect(error).to.not.be.ok();
                 expect(data).to.be.ok();
 
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
+                redisIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
                 expect(redisIp).to.be.ok();
 
                 exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
@@ -1219,37 +1217,6 @@ describe('Apps', function () {
             });
         });
 
-        it('installation - redis addon config', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                var password = urlp.auth.split(':')[1];
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                function checkRedis() {
-                    var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                    client.on('error', done);
-                    client.set('key', 'value');
-                    client.get('key', function (err, reply) {
-                        expect(err).to.not.be.ok();
-                        expect(reply.toString()).to.be('value');
-                        client.end();
-                        done();
-                    });
-                }
-
-                setTimeout(checkRedis, 1000); // the bridge network takes time to come up?
-            });
-        });
-
         function checkConfigureStatus(count, done) {
             assert.strictEqual(typeof count, 'number');
             assert.strictEqual(typeof done, 'function');
@@ -1265,20 +1232,20 @@ describe('Apps', function () {
             });
         }
 
-        it('cannot reconfigure app with missing location', function (done) {
+        it('cannot reconfigure app with bad location', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: 1234, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with missing accessRestriction', function (done) {
+        it('cannot reconfigure app with bad accessRestriction', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
+                  .send({ password: PASSWORD, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: false })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1288,7 +1255,7 @@ describe('Apps', function () {
         it('cannot reconfigure app with only the cert, no key', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1298,27 +1265,27 @@ describe('Apps', function () {
         it('cannot reconfigure app with only the key, no cert', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with cert not bein a string', function (done) {
+        it('cannot reconfigure app with cert not being a string', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: 1234, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
             });
         });
 
-        it('cannot reconfigure app with key not bein a string', function (done) {
+        it('cannot reconfigure app with key not being a string', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: 1234 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, cert: validCert1, key: 1234 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(400);
                 done();
@@ -1328,7 +1295,7 @@ describe('Apps', function () {
         it('non admin cannot reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token_1 })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(403);
                 done();
@@ -1338,7 +1305,7 @@ describe('Apps', function () {
         it('can reconfigure app', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 } })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 checkConfigureStatus(0, done);
@@ -1351,7 +1318,7 @@ describe('Apps', function () {
                 expect(!error).to.be.ok();
                 expect(app).to.be.an('object');
                 appEntry = app;
-                expect(appEntry.containerid).to.not.be(oldContainerId);
+                expect(appEntry.containerId).to.not.be(oldContainerId);
                 done();
             });
         });
@@ -1372,7 +1339,7 @@ describe('Apps', function () {
                 expect(error).to.not.be.ok();
                 expect(data).to.be.ok();
 
-                redisIp = safe.query(data, 'NetworkSettings.IPAddress');
+                redisIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress');
                 expect(redisIp).to.be.ok();
 
                 exportedRedisPort = safe.query(data, 'NetworkSettings.Ports.6379/tcp[0].HostPort');
@@ -1382,47 +1349,16 @@ describe('Apps', function () {
             });
         });
 
-        it('redis addon works after reconfiguration', function (done) {
-            docker.getContainer(appEntry.containerId).inspect(function (error, data) {
-                var redisUrl = null;
-                data.Config.Env.forEach(function (env) { if (env.indexOf('REDIS_URL=') === 0) redisUrl = env.split('=')[1]; });
-                expect(redisUrl).to.be.ok();
-
-                var urlp = url.parse(redisUrl);
-                var password = urlp.auth.split(':')[1];
-
-                expect(urlp.hostname).to.be('redis-' + APP_ID);
-
-                expect(data.Config.Env).to.contain('REDIS_PORT=6379');
-                expect(data.Config.Env).to.contain('REDIS_HOST=redis-' + APP_ID);
-                expect(data.Config.Env).to.contain('REDIS_PASSWORD=' + password);
-
-                var client = redis.createClient(parseInt(urlp.port, 10), redisIp, { auth_pass: password });
-                client.on('error', done);
-                client.set('key', 'value');
-                client.get('key', function (err, reply) {
-                    expect(err).to.not.be.ok();
-                    expect(reply.toString()).to.be('value');
-                    client.end();
-                    done();
-                });
-            });
-        });
-
-        it('scheduler works after reconfiguration', function (done) {
-            async.retry({ times: 100, interval: 1000 }, function (callback) {
-                var data = safe.fs.readFileSync(paths.DATA_DIR + '/' + APP_ID + '/data/every_minute.env', 'utf8');
-
-                if (data && data.indexOf('ECHO_SERVER_PORT=7172') !== -1) return callback();
-
-                callback(new Error('not run yet'));
-            }, done);
+        it('installation - app can check addons', function (done) {
+            this.timeout(120000);
+            console.log('This test can take a while as it waits for scheduler addon to tick 4');
+            checkAddons(appEntry, done);
         });
 
         it('can reconfigure app with custom certificate', function (done) {
             superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/configure')
                   .query({ access_token: token })
-                  .send({ appId: APP_ID, password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
+                  .send({ password: PASSWORD, location: APP_LOCATION_NEW, portBindings: { ECHO_SERVER_PORT: 7172 }, accessRestriction: null, cert: validCert1, key: validKey1 })
                   .end(function (err, res) {
                 expect(res.statusCode).to.equal(202);
                 checkConfigureStatus(0, done);

src/routes/test/backups-test.js

@@ -53,7 +53,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/clients-test.js

@@ -8,7 +8,7 @@
 
 var async = require('async'),
     config = require('../../config.js'),
-    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     database = require('../../database.js'),
     oauth2 = require('../oauth2.js'),
     expect = require('expect.js'),
@@ -174,7 +174,7 @@ describe('OAuth Clients API', function () {
                     expect(result.body.redirectURI).to.be.a('string');
                     expect(result.body.clientSecret).to.be.a('string');
                     expect(result.body.scope).to.be.a('string');
-                    expect(result.body.type).to.equal(clientdb.TYPE_EXTERNAL);
+                    expect(result.body.type).to.equal(clients.TYPE_EXTERNAL);
 
                     done();
                 });
@@ -291,6 +291,14 @@ describe('OAuth Clients API', function () {
             scope: 'profile'
         };
 
+        var CLIENT_1 = {
+            id: '',
+            appId: 'someAppId-1',
+            redirectURI: 'http://some.callback1',
+            scope: 'profile',
+            type: clients.TYPE_OAUTH
+        };
+
         before(function (done) {
             async.series([
                 server.start.bind(null),
@@ -387,6 +395,44 @@ describe('OAuth Clients API', function () {
                    });
                 });
             });
+
+            it('fails for cid-webadmin', function (done) {
+                superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin')
+                       .query({ access_token: token })
+                       .end(function (error, result) {
+                    expect(result.statusCode).to.equal(405);
+
+                    superagent.get(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin')
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(200);
+
+                        done();
+                   });
+                });
+            });
+
+            it('fails for addon auth client', function (done) {
+                clients.add(CLIENT_1.appId, CLIENT_1.type, CLIENT_1.redirectURI, CLIENT_1.scope, function (error, result) {
+                    expect(error).to.equal(null);
+
+                    CLIENT_1.id = result.id;
+
+                    superagent.del(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_1.id)
+                           .query({ access_token: token })
+                           .end(function (error, result) {
+                        expect(result.statusCode).to.equal(405);
+
+                        superagent.get(SERVER_URL + '/api/v1/oauth/clients/' + CLIENT_1.id)
+                               .query({ access_token: token })
+                               .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+
+                            done();
+                       });
+                    });
+                });
+            });
         });
     });
 });
@@ -489,8 +535,7 @@ describe('Clients', function () {
             .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
 
-                expect(result.body.clients.length).to.eql(1);
-                expect(result.body.clients[0].tokenCount).to.eql(1);
+                expect(result.body.clients.length).to.eql(3);
 
                 done();
             });
@@ -543,7 +588,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
 
                 done();
             });
@@ -596,7 +641,7 @@ describe('Clients', function () {
                 expect(result.statusCode).to.equal(200);
 
                 expect(result.body.tokens.length).to.eql(1);
-                expect(result.body.tokens[0].identifier).to.eql('user-' + USER_0.id);
+                expect(result.body.tokens[0].identifier).to.eql(USER_0.id);
 
                 superagent.del(SERVER_URL + '/api/v1/oauth/clients/cid-webadmin/tokens')
                 .query({ access_token: token })

src/routes/test/developer-test.js

@@ -327,7 +327,7 @@ describe('Developer API', function () {
                    .send({ username: USERNAME, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -338,7 +338,7 @@ describe('Developer API', function () {
                    .send({ username: USERNAME.toUpperCase(), password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -349,7 +349,7 @@ describe('Developer API', function () {
                    .send({ username: EMAIL, password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
@@ -360,10 +360,66 @@ describe('Developer API', function () {
                    .send({ username: EMAIL.toUpperCase(), password: PASSWORD })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(200);
-                expect(result.body.expiresAt).to.be.a('number');
+                expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
                 expect(result.body.token).to.be.a('string');
                 done();
             });
         });
     });
+
+    describe('sdk tokens are valid without password checks', function () {
+        var token_normal, token_sdk;
+
+        before(function (done) {
+            async.series([
+                setup,
+
+                settings.setDeveloperMode.bind(null, true),
+
+                function (callback) {
+                    var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+                    var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+                    superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                           .query({ setupToken: 'somesetuptoken' })
+                           .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                           .end(function (error, result) {
+                        expect(result).to.be.ok();
+                        expect(scope1.isDone()).to.be.ok();
+                        expect(scope2.isDone()).to.be.ok();
+
+                        token_normal = result.body.token;
+
+                        superagent.post(SERVER_URL + '/api/v1/developer/login')
+                          .send({ username: USERNAME, password: PASSWORD })
+                          .end(function (error, result) {
+                            expect(result.statusCode).to.equal(200);
+                            expect(new Date(result.body.expiresAt).toString()).to.not.be('Invalid Date');
+                            expect(result.body.token).to.be.a('string');
+
+                            token_sdk = result.body.token;
+
+                            callback();
+                        });
+                    });
+                },
+            ], done);
+        });
+
+        after(cleanup);
+
+        it('fails with non sdk token', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_normal }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(400);
+                done();
+            });
+        });
+
+        it('succeeds', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/profile/password').query({ access_token: token_sdk }).send({ newPassword: 'Some?$123' }).end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+    });
 });

src/routes/test/eventlog-test.js

@@ -67,7 +67,7 @@ function setup(done) {
             token_1 = tokendb.generateToken();
 
             // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-            tokendb.add(token_1, tokendb.PREFIX_USER + USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
+            tokendb.add(token_1, USER_1_ID, 'test-client-id',  Date.now() + 100000, '*', callback);
         }
 
     ], done);

src/routes/test/groups-test.js

@@ -6,18 +6,15 @@
 
 'use strict';
 
-var appdb = require('../../appdb.js'),
-    async = require('async'),
+var async = require('async'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
     groups = require('../../groups.js'),
     superagent = require('superagent'),
     server = require('../../server.js'),
-    settings = require('../../settings.js'),
     tokendb = require('../../tokendb.js'),
-    nock = require('nock'),
-    userdb = require('../../userdb.js');
+    nock = require('nock');
 
 var SERVER_URL = 'http://localhost:' + config.get('port');
 
@@ -117,6 +114,9 @@ describe('Groups API', function () {
                 expect(res.body.groups).to.be.an(Array);
                 expect(res.body.groups.length).to.be(1);
                 expect(res.body.groups[0].name).to.eql('admin');
+                expect(res.body.groups[0].userIds).to.be.an(Array);
+                expect(res.body.groups[0].userIds.length).to.be(1);
+                expect(res.body.groups[0].userIds[0]).to.be(userId);
                 done();
             });
         });
@@ -224,7 +224,7 @@ describe('Groups API', function () {
         });
 
         it('cannot add user to invalid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'something' ]})
                   .end(function (error, result) {
@@ -234,7 +234,7 @@ describe('Groups API', function () {
         });
 
         it('can add user to valid group', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'admin', 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -243,8 +243,8 @@ describe('Groups API', function () {
             });
         });
 
-        it('can remove last user from admin', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/set_groups')
+        it('cannot remove self from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
                   .query({ access_token: token })
                   .send({ groupIds: [ 'group0', 'group1' ]})
                   .end(function (error, result) {
@@ -252,5 +252,37 @@ describe('Groups API', function () {
                 done();
             });
         });
+
+        it('can add another user to admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId_1 + '/groups')
+                  .query({ access_token: token })
+                  .send({ groupIds: [ 'admin' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204);
+                done();
+            });
+        });
+
+        it('lists members of admin group', function (done) {
+            superagent.get(SERVER_URL + '/api/v1/groups/admin')
+                  .query({ access_token: token })
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(200);
+                expect(result.body.userIds.length).to.be(2);
+                expect(result.body.userIds[0]).to.be(userId);
+                expect(result.body.userIds[1]).to.be(userId_1);
+                done();
+            });
+        });
+
+        it('remove activation user from admin', function (done) {
+            superagent.put(SERVER_URL + '/api/v1/users/' + userId + '/groups')
+                  .query({ access_token: token_1 })
+                  .send({ groupIds: [ 'group0', 'group1' ]})
+                  .end(function (error, result) {
+                expect(result.statusCode).to.equal(204); // user_1 is still admin, so we can remove the other person
+                done();
+            });
+        });
     });
 });

src/routes/test/mailboxes-test.js

@@ -0,0 +1,211 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    superagent = require('superagent'),
+    server = require('../../server.js'),
+    nock = require('nock'),
+    userdb = require('../../userdb.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var MAILBOX_ID = 'mailbox';
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.set('fqdn', 'foobar.com');
+
+    async.series([
+        server.start.bind(server),
+
+        userdb._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('Mailbox API', function () {
+    this.timeout(10000);
+
+    before(setup);
+    after(cleanup);
+
+    it('cannot create a mailbox without name param', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot create a mailbox without token', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(401);
+            done();
+        });
+    });
+
+    it('cannot create invalid mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: 'no-reply' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can create mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('can get mailbox', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.name).to.equal(MAILBOX_ID);
+            expect(res.body.creationTime).to.be.ok();
+            done();
+        });
+    });
+
+    it('cannot set with invalid alias', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'a' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('cannot set with invalid type', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'apple', 34 ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('can set aliases of mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ 'alias1', 'alias2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            done();
+        });
+    });
+
+    it('can list mailboxes', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.mailboxes).to.be.an(Array);
+            expect(res.body.mailboxes[0].name).to.be(MAILBOX_ID);
+            done();
+        });
+    });
+
+    it('can get aliases', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.aliases).to.be.an(Array);
+            expect(res.body.aliases[0]).to.be('alias1');
+            expect(res.body.aliases[1]).to.be('alias2');
+            done();
+        });
+    });
+
+    it('can add another mailbox', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/mailboxes')
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID + '2' })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(201);
+            done();
+        });
+    });
+
+    it('cannot alias existing mailbox', function (done) {
+        superagent.put(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID + '/aliases')
+               .query({ access_token: token })
+               .send({ aliases: [ MAILBOX_ID + '2' ]})
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(409);
+            done();
+        });
+    });
+
+    it('can delete mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(204);
+            done();
+        });
+    });
+
+    it('cannot delete random mailbox', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/mailboxes/' + MAILBOX_ID)
+               .query({ access_token: token })
+               .send({ name: MAILBOX_ID })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(404);
+            done();
+        });
+    });
+});

src/routes/test/oauth2-test.js

@@ -18,6 +18,7 @@ var expect = require('expect.js'),
     querystring = require('querystring'),
     database = require('../../database.js'),
     clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     userdb = require('../../userdb.js'),
     user = require('../../user.js'),
     appdb = require('../../appdb.js'),
@@ -197,7 +198,7 @@ describe('OAuth2', function () {
         var CLIENT_0 = {
             id: 'cid-client0',
             appId: 'appid-app0',
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret0',
             redirectURI: 'http://redirect0',
             scope: 'profile'
@@ -207,7 +208,7 @@ describe('OAuth2', function () {
         var CLIENT_1 = {
             id: 'cid-client1',
             appId: 'appid-app1',
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret1',
             redirectURI: 'http://redirect1',
             scope: 'profile'
@@ -217,7 +218,7 @@ describe('OAuth2', function () {
         var CLIENT_2 = {
             id: 'cid-client2',
             appId: APP_0.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret2',
             redirectURI: 'http://redirect2',
             scope: 'profile'
@@ -227,7 +228,7 @@ describe('OAuth2', function () {
         var CLIENT_3 = {
             id: 'cid-client3',
             appId: APP_0.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret3',
             redirectURI: 'http://redirect1',
             scope: 'profile'
@@ -237,7 +238,7 @@ describe('OAuth2', function () {
         var CLIENT_4 = {
             id: 'cid-client4',
             appId: 'appid-app4',
-            type: clientdb.TYPE_PROXY,
+            type: clients.TYPE_PROXY,
             clientSecret: 'secret4',
             redirectURI: 'http://redirect4',
             scope: 'profile'
@@ -247,7 +248,7 @@ describe('OAuth2', function () {
         var CLIENT_5 = {
             id: 'cid-client5',
             appId: APP_0.id,
-            type: clientdb.TYPE_PROXY,
+            type: clients.TYPE_PROXY,
             clientSecret: 'secret5',
             redirectURI: 'http://redirect5',
             scope: 'profile'
@@ -257,7 +258,7 @@ describe('OAuth2', function () {
         var CLIENT_6 = {
             id: 'cid-client6',
             appId: APP_1.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret6',
             redirectURI: 'http://redirect6',
             scope: 'profile'
@@ -267,7 +268,7 @@ describe('OAuth2', function () {
         var CLIENT_7 = {
             id: 'cid-client7',
             appId: APP_2.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret7',
             redirectURI: 'http://redirect7',
             scope: 'profile'
@@ -277,7 +278,7 @@ describe('OAuth2', function () {
         var CLIENT_8 = {
             id: 'cid-client8',
             appId: APP_2.id,
-            type: clientdb.TYPE_SIMPLE_AUTH,
+            type: clients.TYPE_SIMPLE_AUTH,
             clientSecret: 'secret8',
             redirectURI: 'http://redirect8',
             scope: 'profile'
@@ -287,7 +288,7 @@ describe('OAuth2', function () {
         var CLIENT_9 = {
             id: 'cid-client9',
             appId: APP_3.id,
-            type: clientdb.TYPE_OAUTH,
+            type: clients.TYPE_OAUTH,
             clientSecret: 'secret9',
             redirectURI: 'http://redirect9',
             scope: 'profile'
@@ -313,10 +314,10 @@ describe('OAuth2', function () {
                 clientdb.add.bind(null, CLIENT_7.id, CLIENT_7.appId, CLIENT_7.type, CLIENT_7.clientSecret, CLIENT_7.redirectURI, CLIENT_7.scope),
                 clientdb.add.bind(null, CLIENT_8.id, CLIENT_8.appId, CLIENT_8.type, CLIENT_8.clientSecret, CLIENT_8.redirectURI, CLIENT_8.scope),
                 clientdb.add.bind(null, CLIENT_9.id, CLIENT_9.appId, CLIENT_9.type, CLIENT_9.clientSecret, CLIENT_9.redirectURI, CLIENT_9.scope),
-                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, APP_0.altDomain),
-                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, APP_1.altDomain),
-                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, APP_2.altDomain),
-                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit, APP_3.altDomain),
+                appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+                appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+                appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2),
+                appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3),
                 function (callback) {
                     user.create(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, null /* source */, function (error, userObject) {
                         expect(error).to.not.be.ok();

src/routes/test/profile-test.js

@@ -115,7 +115,7 @@ describe('Profile API', function () {
             var token = tokendb.generateToken();
             var expires = Date.now() - 2000; // 1 sec
 
-            tokendb.add(token, tokendb.PREFIX_USER + user_0.id, null, expires, '*', function (error) {
+            tokendb.add(token, user_0.id, null, expires, '*', function (error) {
                 expect(error).to.not.be.ok();
 
                 superagent.get(SERVER_URL + '/api/v1/profile').query({ access_token: token }).end(function (error, result) {
@@ -153,7 +153,7 @@ describe('Profile API', function () {
         after(cleanup);
 
         it('change email fails due to missing token', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .send({ email: EMAIL_0_NEW })
                    .end(function (error, result) {
                 expect(result.statusCode).to.equal(401);
@@ -162,7 +162,7 @@ describe('Profile API', function () {
         });
 
         it('change email fails due to invalid email', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({ email: 'foo@bar' })
                    .end(function (error, result) {
@@ -172,7 +172,7 @@ describe('Profile API', function () {
         });
 
         it('change user succeeds without email nor displayName', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({})
                    .end(function (error, result) {
@@ -182,7 +182,7 @@ describe('Profile API', function () {
         });
 
         it('change email succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({ email: EMAIL_0_NEW })
                    .end(function (error, result) {
@@ -203,7 +203,7 @@ describe('Profile API', function () {
         });
 
         it('change displayName succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile')
+            superagent.post(SERVER_URL + '/api/v1/profile')
                    .query({ access_token: token_0 })
                    .send({ displayName: DISPLAY_NAME_0_NEW })
                    .end(function (error, result) {
@@ -229,7 +229,7 @@ describe('Profile API', function () {
         after(cleanup);
 
         it('fails due to missing current password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ newPassword: 'some wrong password' })
                    .end(function (err, res) {
@@ -239,7 +239,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to missing new password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: PASSWORD })
                    .end(function (err, res) {
@@ -249,7 +249,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to wrong password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: 'some wrong password', newPassword: 'MOre#$%34' })
                    .end(function (err, res) {
@@ -259,7 +259,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to invalid password', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: PASSWORD, newPassword: 'five' })
                    .end(function (err, res) {
@@ -269,7 +269,7 @@ describe('Profile API', function () {
         });
 
         it('succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/password')
+            superagent.post(SERVER_URL + '/api/v1/profile/password')
                    .query({ access_token: token_0 })
                    .send({ password: PASSWORD, newPassword: 'MOre#$%34' })
                    .end(function (err, res) {
@@ -284,7 +284,7 @@ describe('Profile API', function () {
         after(cleanup);
 
         it('fails due to missing showTutorial', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/tutorial')
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
                    .query({ access_token: token_0 })
                    .send({})
                    .end(function (err, res) {
@@ -294,7 +294,7 @@ describe('Profile API', function () {
         });
 
         it('fails due to wrong showTutorial type', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/tutorial')
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
                    .query({ access_token: token_0 })
                    .send({ showTutorial: 'true' })
                    .end(function (err, res) {
@@ -304,7 +304,7 @@ describe('Profile API', function () {
         });
 
         it('succeeds', function (done) {
-            superagent.put(SERVER_URL + '/api/v1/profile/tutorial')
+            superagent.post(SERVER_URL + '/api/v1/profile/tutorial')
                    .query({ access_token: token_0 })
                    .send({ showTutorial: false })
                    .end(function (err, res) {

src/routes/test/server-test.js

@@ -0,0 +1,88 @@
+/* jslint node:true */
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var async = require('async'),
+    config = require('../../config.js'),
+    database = require('../../database.js'),
+    expect = require('expect.js'),
+    net = require('net'),
+    nock = require('nock'),
+    superagent = require('superagent'),
+    server = require('../../server.js');
+
+var SERVER_URL = 'http://localhost:' + config.get('port');
+
+var USERNAME = 'superadmin', PASSWORD = 'Foobar?1337', EMAIL ='silly@me.com';
+var token = null;
+
+var server;
+function setup(done) {
+    config.setVersion('1.2.3');
+
+    async.series([
+        server.start.bind(server),
+
+        database._clear,
+
+        function createAdmin(callback) {
+            var scope1 = nock(config.apiServerOrigin()).get('/api/v1/boxes/' + config.fqdn() + '/setup/verify?setupToken=somesetuptoken').reply(200, {});
+            var scope2 = nock(config.apiServerOrigin()).post('/api/v1/boxes/' + config.fqdn() + '/setup/done?setupToken=somesetuptoken').reply(201, {});
+
+            superagent.post(SERVER_URL + '/api/v1/cloudron/activate')
+                   .query({ setupToken: 'somesetuptoken' })
+                   .send({ username: USERNAME, password: PASSWORD, email: EMAIL })
+                   .end(function (error, result) {
+                expect(result).to.be.ok();
+                expect(result.statusCode).to.eql(201);
+                expect(scope1.isDone()).to.be.ok();
+                expect(scope2.isDone()).to.be.ok();
+
+                // stash token for further use
+                token = result.body.token;
+
+                callback();
+            });
+        }
+    ], done);
+}
+
+function cleanup(done) {
+    database._clear(function (error) {
+        expect(!error).to.be.ok();
+
+        server.stop(done);
+    });
+}
+
+describe('REST API', function () {
+    before(setup);
+    after(cleanup);
+
+    it('does not crash with invalid JSON', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+            .query({ access_token: token })
+            .set('content-type', 'application/json')
+            .send("some invalid non-strict json")
+           .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            expect(result.body.message).to.be('Bad JSON');
+            done();
+        });
+    });
+
+    it('does not crash with invalid string', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+            .query({ access_token: token })
+            .set('content-type', 'application/x-www-form-urlencoded')
+            .send("some string")
+           .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+});

src/routes/test/settings-test.js

@@ -56,7 +56,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok' };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         }
     ], done);
 }

src/routes/test/simpleauth-test.js

@@ -6,9 +6,10 @@
 
 'use strict';
 
-var clientdb = require('../../clientdb.js'),
-    appdb = require('../../appdb.js'),
+var appdb = require('../../appdb.js'),
     async = require('async'),
+    clientdb = require('../../clientdb.js'),
+    clients = require('../../clients.js'),
     config = require('../../config.js'),
     database = require('../../database.js'),
     expect = require('expect.js'),
@@ -70,7 +71,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_0 = {
         id: 'someclientid',
         appId: 'someappid',
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret',
         redirectURI: '',
         scope: 'user,profile'
@@ -79,7 +80,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_1 = {
         id: 'someclientid1',
         appId: APP_0.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret1',
         redirectURI: '',
         scope: 'user,profile'
@@ -88,7 +89,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_2 = {
         id: 'someclientid2',
         appId: APP_1.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret2',
         redirectURI: '',
         scope: 'user,profile'
@@ -97,7 +98,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_3 = {
         id: 'someclientid3',
         appId: APP_2.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret3',
         redirectURI: '',
         scope: 'user,profile'
@@ -106,7 +107,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_4 = {
         id: 'someclientid4',
         appId: APP_2.id,
-        type: clientdb.TYPE_OAUTH,
+        type: clients.TYPE_OAUTH,
         clientSecret: 'someclientsecret4',
         redirectURI: '',
         scope: 'user,profile'
@@ -115,7 +116,7 @@ describe('SimpleAuth API', function () {
     var CLIENT_5 = {
         id: 'someclientid5',
         appId: APP_3.id,
-        type: clientdb.TYPE_SIMPLE_AUTH,
+        type: clients.TYPE_SIMPLE_AUTH,
         clientSecret: 'someclientsecret5',
         redirectURI: '',
         scope: 'user,profile'
@@ -159,10 +160,10 @@ describe('SimpleAuth API', function () {
             clientdb.add.bind(null, CLIENT_3.id, CLIENT_3.appId, CLIENT_3.type, CLIENT_3.clientSecret, CLIENT_3.redirectURI, CLIENT_3.scope),
             clientdb.add.bind(null, CLIENT_4.id, CLIENT_4.appId, CLIENT_4.type, CLIENT_4.clientSecret, CLIENT_4.redirectURI, CLIENT_4.scope),
             clientdb.add.bind(null, CLIENT_5.id, CLIENT_5.appId, CLIENT_5.type, CLIENT_5.clientSecret, CLIENT_5.redirectURI, CLIENT_5.scope),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, APP_0.altDomain),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, APP_1.altDomain),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, APP_2.altDomain),
-            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3.accessRestriction, APP_3.memoryLimit, APP_3.altDomain)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2),
+            appdb.add.bind(null, APP_3.id, APP_3.appStoreId, APP_3.manifest, APP_3.location, APP_3.portBindings, APP_3)
         ], done);
     });
 

src/routes/test/start_addons.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
-
-source ${SOURCE_DIR}/src/INFRA_VERSION
-
-readonly mysqldatadir="/tmp/mysqldata-$(date +%s)"
-readonly postgresqldatadir="/tmp/postgresqldata-$(date +%s)"
-readonly mongodbdatadir="/tmp/mongodbdata-$(date +%s)"
-root_password=secret
-
-start_postgresql() {
-    postgresql_vars="POSTGRESQL_ROOT_PASSWORD=${root_password}; POSTGRESQL_ROOT_HOST=172.17.0.0/255.255.0.0"
-
-    rm -rf /tmp/postgresql_vars.sh
-    echo "${postgresql_vars}" > /tmp/postgresql_vars.sh
-
-    docker rm -f postgresql 2>/dev/null 1>&2 || true
-
-    docker run -dtP --name=postgresql -v "${postgresqldatadir}:/var/lib/postgresql" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh "${POSTGRESQL_IMAGE}" >/dev/null
-}
-
-start_mysql() {
-    local mysql_vars="MYSQL_ROOT_PASSWORD=${root_password}; MYSQL_ROOT_HOST=172.17.0.0/255.255.0.0"
-
-    rm -rf /tmp/mysql_vars.sh
-    echo "${mysql_vars}" > /tmp/mysql_vars.sh
-
-    docker rm -f mysql 2>/dev/null 1>&2 || true
-
-    docker run -dP --name=mysql -v "${mysqldatadir}:/var/lib/mysql" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/mysql_vars.sh:/etc/mysql/mysql_vars.sh "${MYSQL_IMAGE}" >/dev/null
-}
-
-start_mongodb() {
-    local mongodb_vars="MONGODB_ROOT_PASSWORD=${root_password}"
-
-    rm -rf /tmp/mongodb_vars.sh
-    echo "${mongodb_vars}" > /tmp/mongodb_vars.sh
-
-    docker rm -f mongodb 2>/dev/null 1>&2 || true
-
-    docker run -dP --name=mongodb -v "${mongodbdatadir}:/var/lib/mongodb" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/mongodb_vars.sh:/etc/mongodb_vars.sh "${MONGODB_IMAGE}" >/dev/null
-}
-
-start_mail() {
-    docker rm -f mail 2>/dev/null 1>&2 || true
-
-    docker run -dP --name=mail -e MAIL_SERVER_NAME="server.local" -e MAIL_DOMAIN="server.local" \
-        --read-only -v /tmp -v /run \
-        -v /tmp/maildata:/app/data "${MAIL_IMAGE}" >/dev/null
-}
-
-start_mysql
-start_postgresql
-start_mongodb
-start_mail
-
-echo -n "Waiting for addons to start"
-for i in {1..10}; do
-    echo -n "."
-    sleep 1
-done
-echo ""
-

src/routes/test/sysadmin-test.js

@@ -51,7 +51,7 @@ function setup(done) {
 
         function addApp(callback) {
             var manifest = { version: '0.0.1', manifestVersion: 1, dockerImage: 'foo', healthCheckPath: '/', httpPort: 3, title: 'ok', addons: { } };
-            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, null /* accessRestriction */, 0 /* memoryLimit */, null /* altDomain */, callback);
+            appdb.add('appid', 'appStoreId', manifest, 'location', [ ] /* portBindings */, { }, callback);
         },
 
         function createSettings(callback) {

src/routes/test/user-test.js

@@ -1,4 +1,3 @@
-/* jslint node:true */
 /* global it:false */
 /* global describe:false */
 /* global before:false */
@@ -21,7 +20,7 @@ var SERVER_URL = 'http://localhost:' + config.get('port');
 var USERNAME_0 = 'superaDmIn', PASSWORD = 'Foobar?1337', EMAIL_0 = 'silLY@me.com', EMAIL_0_NEW = 'stupID@me.com', DISPLAY_NAME_0_NEW = 'New Name';
 var USERNAME_1 = 'userTheFirst', EMAIL_1 = 'taO@zen.mac';
 var USERNAME_2 = 'userTheSecond', EMAIL_2 = 'USER@foo.bar', EMAIL_2_NEW = 'happy@ME.com';
-var USERNAME_3 = 'userTheThird', EMAIL_3 = 'user3@FOO.bar';
+var USERNAME_3 = 'ut', EMAIL_3 = 'user3@FOO.bar';
 
 function setup(done) {
     server.start(function (error) {
@@ -161,7 +160,7 @@ describe('User API', function () {
         var token = tokendb.generateToken();
         var expires = Date.now() + 2000; // 1 sec
 
-        tokendb.add(token, tokendb.PREFIX_USER + user_0.id, null, expires, '*', function (error) {
+        tokendb.add(token, user_0.id, null, expires, '*', function (error) {
             expect(error).to.not.be.ok();
 
             setTimeout(function () {
@@ -261,7 +260,7 @@ describe('User API', function () {
 
             checkMails(2, function () {
               // HACK to get a token for second user (passwords are generated and the user should have gotten a password setup link...)
-              tokendb.add(token_1, tokendb.PREFIX_USER + user_1.id, 'test-client-id',  Date.now() + 10000, '*', done);
+              tokendb.add(token_1, user_1.id, 'test-client-id',  Date.now() + 10000, '*', done);
             });
         });
     });
@@ -293,7 +292,7 @@ describe('User API', function () {
     });
 
     it('set second user as admin succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/groups')
                .query({ access_token: token })
                .send({ groupIds: [ groups.ADMIN_GROUP_ID ] })
                .end(function (err, res) {
@@ -310,8 +309,24 @@ describe('User API', function () {
         });
     });
 
+    it('list groupIds when listing users', function (done) {
+        superagent.get(SERVER_URL + '/api/v1/users')
+        .query({ access_token: token })
+        .end(function (error, res) {
+            expect(error).to.be(null);
+            expect(res.statusCode).to.equal(200);
+            expect(res.body.users).to.be.an('array');
+
+            res.body.users.forEach(function (user) {
+                expect(user.admin).to.be(true);
+                expect(user.groupIds).to.eql([ groups.ADMIN_GROUP_ID ]);
+            });
+            done();
+        });
+    });
+
     it('remove itself from admins fails', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id + '/groups')
                .query({ access_token: token })
                .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
@@ -321,7 +336,7 @@ describe('User API', function () {
     });
 
     it('remove second user from admins succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/set_groups')
+        superagent.put(SERVER_URL + '/api/v1/users/' + user_1.id + '/groups')
                .query({ access_token: token })
                .send({ groupIds: [ 'somegroupid' ] })
                .end(function (err, res) {
@@ -368,6 +383,26 @@ describe('User API', function () {
         });
     });
 
+    it('create user reserved name fails', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+               .query({ access_token: token })
+               .send({ username: 'no-reply' })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+
+    it('create user with short name fails', function (done) {
+        superagent.post(SERVER_URL + '/api/v1/users')
+               .query({ access_token: token })
+               .send({ username: 'n' })
+               .end(function (error, result) {
+            expect(result.statusCode).to.equal(400);
+            done();
+        });
+    });
+
     it('create second and third user', function (done) {
         mailer._clearMailQueue();
 
@@ -441,12 +476,24 @@ describe('User API', function () {
                 expect(user.email).to.be.ok();
                 expect(user.password).to.not.be.ok();
                 expect(user.salt).to.not.be.ok();
+                expect(user.groupIds).to.be.an(Array);
+                expect(user.admin).to.be.a('boolean');
             });
 
             done();
         });
     });
 
+    it('remove random user fails', function (done) {
+        superagent.del(SERVER_URL + '/api/v1/users/randomid')
+               .query({ access_token: token })
+               .send({ password: PASSWORD })
+               .end(function (err, res) {
+            expect(res.statusCode).to.equal(404);
+            done();
+        });
+    });
+
     it('user removes himself is not allowed', function (done) {
         superagent.del(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
@@ -508,7 +555,7 @@ describe('User API', function () {
 
     // Change email
     it('change email fails due to missing token', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .send({ email: EMAIL_0_NEW })
                .end(function (error, result) {
             expect(result.statusCode).to.equal(401);
@@ -517,7 +564,7 @@ describe('User API', function () {
     });
 
     it('change email fails due to invalid email', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ email: 'foo@bar' })
                .end(function (error, result) {
@@ -527,7 +574,7 @@ describe('User API', function () {
     });
 
     it('change user succeeds without email nor displayName', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({})
                .end(function (error, result) {
@@ -537,7 +584,7 @@ describe('User API', function () {
     });
 
     it('change email succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_2.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_2.id)
                .query({ access_token: token })
                .send({ email: EMAIL_2_NEW })
                .end(function (error, result) {
@@ -558,7 +605,7 @@ describe('User API', function () {
     });
 
     it('change email as admin for other user succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_2.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_2.id)
                .query({ access_token: token })
                .send({ email: EMAIL_2 })
                .end(function (error, result) {
@@ -579,7 +626,7 @@ describe('User API', function () {
     });
 
     it('change displayName succeeds', function (done) {
-        superagent.put(SERVER_URL + '/api/v1/users/' + user_0.id)
+        superagent.post(SERVER_URL + '/api/v1/users/' + user_0.id)
                .query({ access_token: token })
                .send({ displayName: DISPLAY_NAME_0_NEW })
                .end(function (error, result) {

src/routes/user.js

@@ -13,13 +13,16 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
+    clients = require('../clients.js'),
     generatePassword = require('../password.js').generate,
     groups = require('../groups.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    oauth2 = require('./oauth2.js'),
     user = require('../user.js'),
     tokendb = require('../tokendb.js'),
-    UserError = user.UserError;
+    UserError = user.UserError,
+    _ = require('underscore');
 
 function auditSource(req) {
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || null;
@@ -41,11 +44,8 @@ function create(req, res, next) {
     var displayName = req.body.displayName || '';
 
     user.create(username, password, email, displayName, auditSource(req), { invitor: req.user, sendInvite: sendInvite }, function (error, user) {
-        if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, 'Invalid username'));
-        if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, 'Invalid email'));
-        if (error && error.reason === UserError.BAD_PASSWORD) return next(new HttpError(400, 'Invalid password'));
         if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
-        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
         if (error) return next(new HttpError(500, error));
 
         var userInfo = {
@@ -54,6 +54,7 @@ function create(req, res, next) {
             displayName: user.displayName,
             email: user.email,
             admin: user.admin,
+            groupIds: [ ],
             resetToken: user.resetToken
         };
 
@@ -68,30 +69,29 @@ function update(req, res, next) {
 
     if ('email' in req.body && typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be string'));
     if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be string'));
+    if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a string'));
 
-    if (req.user.tokenType !== tokendb.TYPE_USER) return next(new HttpError(403, 'Token type not allowed'));
     if (req.user.id !== req.params.userId && !req.user.admin) return next(new HttpError(403, 'Not allowed'));
 
-    user.get(req.params.userId, function (error, result) {
-        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+    user.update(req.params.userId, req.body, auditSource(req), function (error) {
+        if (error && error.reason === UserError.BAD_FIELD) return next(new HttpError(400, error.message));
+        if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, error.message));
+        if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
         if (error) return next(new HttpError(500, error));
 
-        user.update(req.params.userId, result.username, req.body.email || result.email, req.body.displayName || result.displayName, auditSource(req), function (error) {
-            if (error && error.reason === UserError.BAD_USERNAME) return next(new HttpError(400, error.message));
-            if (error && error.reason === UserError.BAD_EMAIL) return next(new HttpError(400, error.message));
-            if (error && error.reason === UserError.ALREADY_EXISTS) return next(new HttpError(409, 'Already exists'));
-            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'User not found'));
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(204));
-        });
+        next(new HttpSuccess(204));
     });
 }
 
 function list(req, res, next) {
-    user.list(function (error, result) {
+    user.list(function (error, results) {
         if (error) return next(new HttpError(500, error));
-        next(new HttpSuccess(200, { users: result }));
+
+        var users = results.map(function (result) {
+            return _.pick(result, 'id', 'username', 'email', 'displayName', 'groupIds', 'admin');
+        });
+
+        next(new HttpSuccess(200, { users: users }));
     });
 }
 
@@ -105,17 +105,14 @@ function get(req, res, next) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        groups.isMember(groups.ADMIN_GROUP_ID, req.params.userId, function (error, isAdmin) {
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(200, {
-                id: result.id,
-                username: result.username,
-                email: result.email,
-                admin: isAdmin,
-                displayName: result.displayName
-            }));
-        });
+        next(new HttpSuccess(200, {
+            id: result.id,
+            username: result.username,
+            displayName: result.displayName,
+            email: result.email,
+            admin: result.admin,
+            groupIds: result.groupIds
+        }));
     });
 }
 
@@ -129,24 +126,20 @@ function remove(req, res, next) {
 
     if (req.user.id === req.params.userId) return next(new HttpError(403, 'Not allowed to remove yourself.'));
 
-    user.get(req.params.userId, function (error, userObject) {
+    user.remove(req.params.userId, auditSource(req), function (error) {
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
         if (error) return next(new HttpError(500, error));
 
-        user.remove(userObject, auditSource(req), function (error) {
-            if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
-            if (error) return next(new HttpError(500, error));
-
-            next(new HttpSuccess(204));
-        });
+        next(new HttpSuccess(204));
     });
 }
 
 function verifyPassword(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    // developers are allowed through without password
-    if (req.user.tokenType === tokendb.TYPE_DEV) return next();
+    // using an 'sdk' token we skip password checks
+    var error = oauth2.validateRequestedScopes(req, [ clients.SCOPE_ROLE_SDK ]);
+    if (!error) return next();
 
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'API call requires user password'));
 

src/scripts/reloadcollectd.sh

@@ -13,6 +13,16 @@ if [[ $# == 1 && "$1" == "--check" ]]; then
 fi
 
 if [[ "${BOX_ENV}" == "cloudron" ]]; then
-    /etc/init.d/collectd restart
+    # when restoring the cloudron with many apps, the apptasks rush in to restart
+    # collectd which makes systemd/collectd very unhappy and puts the collectd in
+    # inactive state
+    for i in {1..10}; do
+        echo "Restarting collectd"
+        if systemctl restart collectd; then
+            exit 0
+        fi
+        echo "Failed to reload collectd. Maybe some other apptask is restarting it"
+        sleep $((RANDOM%30))
+    done
 fi
 

src/scripts/setup_infra.sh

@@ -1,148 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-if [[ ${EUID} -ne 0 ]]; then
-    echo "This script should be run as root." > /dev/stderr
-    exit 1
-fi
-
-if [[ $# == 1 && "$1" == "--check" ]]; then
-    echo "OK"
-    exit 0
-fi
-
-readonly DATA_DIR="/home/yellowtent/data"
-
-script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source "${script_dir}/../INFRA_VERSION" # this injects INFRA_VERSION
-
-readonly fqdn="$1"
-readonly mail_fqdn="$2"
-readonly mail_tls_cert="$3"
-readonly mail_tls_key="$4"
-
-# removing containers ensures containers are launched with latest config updates
-# restore code in appatask does not delete old containers
-infra_version="none"
-[[ -f "${DATA_DIR}/INFRA_VERSION" ]] && infra_version=$(cat "${DATA_DIR}/INFRA_VERSION")
-if [[ "${infra_version}" == "${INFRA_VERSION}" ]]; then
-    echo "Infrastructure is upto date"
-    exit 0
-fi
-
-echo "Upgrading infrastructure from ${infra_version} to ${INFRA_VERSION}"
-
-# TODO: be nice and stop addons cleanly (example, shutdown commands)
-existing_containers=$(docker ps -qa)
-echo "Remove containers: ${existing_containers}"
-if [[ -n "${existing_containers}" ]]; then
-    echo "${existing_containers}" | xargs docker rm -f
-fi
-
-# graphite
-graphite_container_id=$(docker run --restart=always -d --name="graphite" \
-    -m 75m \
-    --memory-swap 150m \
-    -p 127.0.0.1:2003:2003 \
-    -p 127.0.0.1:2004:2004 \
-    -p 127.0.0.1:8000:8000 \
-    -v "${DATA_DIR}/graphite:/app/data" \
-    --read-only -v /tmp -v /run \
-    "${GRAPHITE_IMAGE}")
-echo "Graphite container id: ${graphite_container_id}"
-if docker images "${GRAPHITE_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${GRAPHITE_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old graphite images"
-fi
-
-# mail (MAIL_SMTP_PORT is 2500 in addons.js. used in mailer.js as well)
-# MAIL_SERVER_NAME is the hostname of the mailserver i.e server uses these certs
-# MAIL_DOMAIN is the domain for which this server is relaying mails
-mail_container_id=$(docker run --restart=always -d --name="mail" \
-    -m 75m \
-    --memory-swap 150m \
-    -h "${fqdn}" \
-    -e "MAIL_DOMAIN=${fqdn}" \
-    -e "MAIL_SERVER_NAME=${mail_fqdn}" \
-    -v "${DATA_DIR}/box/mail:/app/data" \
-    -v "${mail_tls_key}:/etc/tls_key.pem:ro" \
-    -v "${mail_tls_cert}:/etc/tls_cert.pem:ro" \
-    --read-only -v /tmp -v /run \
-    "${MAIL_IMAGE}")
-echo "Mail container id: ${mail_container_id}"
-if docker images "${MAIL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MAIL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mail images"
-fi
-
-# mysql
-mysql_addon_root_password=$(pwgen -1 -s)
-docker0_ip=$(/sbin/ifconfig docker0 | grep "inet addr" | awk -F: '{print $2}' | awk '{print $1}')
-cat > "${DATA_DIR}/addons/mysql_vars.sh" <<EOF
-readonly MYSQL_ROOT_PASSWORD='${mysql_addon_root_password}'
-readonly MYSQL_ROOT_HOST='${docker0_ip}'
-EOF
-mysql_container_id=$(docker run --restart=always -d --name="mysql" \
-    -m 256m \
-    --memory-swap 512m \
-    -h "${fqdn}" \
-    -v "${DATA_DIR}/mysql:/var/lib/mysql" \
-    -v "${DATA_DIR}/addons/mysql_vars.sh:/etc/mysql/mysql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MYSQL_IMAGE}")
-echo "MySQL container id: ${mysql_container_id}"
-if docker images "${MYSQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MYSQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mysql images"
-fi
-
-# postgresql
-postgresql_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/postgresql_vars.sh" <<EOF
-readonly POSTGRESQL_ROOT_PASSWORD='${postgresql_addon_root_password}'
-EOF
-postgresql_container_id=$(docker run --restart=always -d --name="postgresql" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${fqdn}" \
-    -v "${DATA_DIR}/postgresql:/var/lib/postgresql" \
-    -v "${DATA_DIR}/addons/postgresql_vars.sh:/etc/postgresql/postgresql_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${POSTGRESQL_IMAGE}")
-echo "PostgreSQL container id: ${postgresql_container_id}"
-if docker images "${POSTGRESQL_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${POSTGRESQL_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old postgresql images"
-fi
-
-# mongodb
-mongodb_addon_root_password=$(pwgen -1 -s)
-cat > "${DATA_DIR}/addons/mongodb_vars.sh" <<EOF
-readonly MONGODB_ROOT_PASSWORD='${mongodb_addon_root_password}'
-EOF
-mongodb_container_id=$(docker run --restart=always -d --name="mongodb" \
-    -m 100m \
-    --memory-swap 200m \
-    -h "${fqdn}" \
-    -v "${DATA_DIR}/mongodb:/var/lib/mongodb" \
-    -v "${DATA_DIR}/addons/mongodb_vars.sh:/etc/mongodb_vars.sh:ro" \
-    --read-only -v /tmp -v /run \
-    "${MONGODB_IMAGE}")
-echo "Mongodb container id: ${mongodb_container_id}"
-if docker images "${MONGODB_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${MONGODB_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old mongodb images"
-fi
-
-# redis
-if docker images "${REDIS_REPO}" | tail -n +2 | awk '{ print $1 ":" $2 }' | grep -v "${REDIS_IMAGE}" | xargs --no-run-if-empty docker rmi; then
-    echo "Removed old redis images"
-fi
-
-# only touch apps in installed state. any other state is just resumed by the taskmanager
-if [[ "${infra_version}" == "none" ]]; then
-    # if no existing infra was found (for new, upgraded and restored cloudons), download app backups
-    echo "Marking installed apps for restore"
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_restore", oldConfigJson = NULL WHERE installationState = "installed"' box
-else
-    # if existing infra was found, just mark apps for reconfiguration
-    mysql -u root -ppassword -e 'UPDATE apps SET installationState = "pending_configure", oldConfigJson = NULL  WHERE installationState = "installed"' box
-fi
-
-echo -n "${INFRA_VERSION}" > "${DATA_DIR}/INFRA_VERSION"

src/server.js

@@ -5,8 +5,7 @@ exports = module.exports = {
     stop: stop
 };
 
-var addons = require('./addons.js'),
-    assert = require('assert'),
+var assert = require('assert'),
     async = require('async'),
     auth = require('./auth.js'),
     certificates = require('./certificates.js'),
@@ -22,6 +21,7 @@ var addons = require('./addons.js'),
     middleware = require('./middleware'),
     passport = require('passport'),
     path = require('path'),
+    platform = require('./platform.js'),
     routes = require('./routes/index.js'),
     taskmanager = require('./taskmanager.js');
 
@@ -32,7 +32,7 @@ function initializeExpressSync() {
     var app = express();
     var httpServer = http.createServer(app);
 
-    var QUERY_LIMIT = '10mb', // max size for json and urlencoded queries
+    var QUERY_LIMIT = '1mb', // max size for json and urlencoded queries (see also client_max_body_size in nginx)
         FIELD_LIMIT = 2 * 1024 * 1024; // max fields that can appear in multipart
 
     var REQUEST_TIMEOUT = 10000; // timeout for all requests (see also setTimeout on the httpServer)
@@ -43,6 +43,7 @@ function initializeExpressSync() {
     app.set('views', path.join(__dirname, 'oauth2views'));
     app.set('view options', { layout: true, debug: false });
     app.set('view engine', 'ejs');
+    app.set('json spaces', 2); // pretty json
 
     if (process.env.BOX_ENV !== 'test') app.use(middleware.morgan('Box :method :url :status :response-time ms - :res[content-length]', { immediate: false }));
 
@@ -62,13 +63,13 @@ function initializeExpressSync() {
        .use(middleware.lastMile());
 
     // NOTE: these limits have to be in sync with nginx limits
-    var FILE_SIZE_LIMIT = '1mb', // max file size that can be uploaded
+    var FILE_SIZE_LIMIT = '1mb', // max file size that can be uploaded (see also client_max_body_size in nginx)
         FILE_TIMEOUT = 60 * 1000; // increased timeout for file uploads (1 min)
 
     var multipart = middleware.multipart({ maxFieldsSize: FIELD_LIMIT, limit: FILE_SIZE_LIMIT, timeout: FILE_TIMEOUT });
 
     // scope middleware implicitly also adds bearer token verification
-    var rootScope = routes.oauth2.scope(clients.SCOPE_ROOT);
+    var cloudronScope = routes.oauth2.scope(clients.SCOPE_CLOUDRON);
     var profileScope = routes.oauth2.scope(clients.SCOPE_PROFILE);
     var usersScope = routes.oauth2.scope(clients.SCOPE_USERS);
     var appsScope = routes.oauth2.scope(clients.SCOPE_APPS);
@@ -90,28 +91,29 @@ function initializeExpressSync() {
     router.post('/api/v1/developer/login', routes.developer.enabled, routes.developer.login);
     router.get ('/api/v1/developer/apps', developerScope, routes.developer.enabled, routes.developer.apps);
 
-    // private routes
-    router.get ('/api/v1/cloudron/config', rootScope, routes.cloudron.getConfig);
-    router.post('/api/v1/cloudron/update', rootScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
-    router.post('/api/v1/cloudron/reboot', rootScope, routes.cloudron.reboot);
-    router.get ('/api/v1/cloudron/graphs', rootScope, routes.graphs.getGraphs);
+    // cloudron routes
+    router.get ('/api/v1/cloudron/config', cloudronScope, routes.cloudron.getConfig);
+    router.post('/api/v1/cloudron/update', cloudronScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.cloudron.update);
+    router.post('/api/v1/cloudron/check_for_updates', cloudronScope, routes.user.requireAdmin, routes.cloudron.checkForUpdates);
+    router.post('/api/v1/cloudron/reboot', cloudronScope, routes.cloudron.reboot);
+    router.get ('/api/v1/cloudron/graphs', cloudronScope, routes.graphs.getGraphs);
 
     // feedback
     router.post('/api/v1/cloudron/feedback', usersScope, routes.cloudron.feedback);
 
     // profile api, working off the user behind the provided token
     router.get ('/api/v1/profile', profileScope, routes.profile.get);
-    router.put ('/api/v1/profile', profileScope, routes.profile.update);
-    router.put ('/api/v1/profile/password', profileScope, routes.user.verifyPassword, routes.profile.changePassword);
-    router.put ('/api/v1/profile/tutorial', profileScope, routes.profile.setShowTutorial);
+    router.post('/api/v1/profile', profileScope, routes.profile.update);
+    router.post('/api/v1/profile/password', profileScope, routes.user.verifyPassword, routes.profile.changePassword);
+    router.post('/api/v1/profile/tutorial', profileScope, routes.profile.setShowTutorial);
 
-    // user routes only for admins
+    // user routes
     router.get ('/api/v1/users', usersScope, routes.user.requireAdmin, routes.user.list);
     router.post('/api/v1/users', usersScope, routes.user.requireAdmin, routes.user.create);
     router.get ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.get);
     router.del ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.user.remove);
-    router.put ('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.update);
-    router.put ('/api/v1/users/:userId/set_groups', usersScope, routes.user.requireAdmin, routes.user.setGroups);
+    router.post('/api/v1/users/:userId', usersScope, routes.user.requireAdmin, routes.user.update);
+    router.put ('/api/v1/users/:userId/groups', usersScope, routes.user.requireAdmin, routes.user.setGroups);
     router.post('/api/v1/users/:userId/invite', usersScope, routes.user.requireAdmin, routes.user.sendInvite);
 
     // Group management
@@ -120,6 +122,14 @@ function initializeExpressSync() {
     router.get ('/api/v1/groups/:groupId', usersScope, routes.user.requireAdmin, routes.groups.get);
     router.del ('/api/v1/groups/:groupId', usersScope, routes.user.requireAdmin, routes.user.verifyPassword, routes.groups.remove);
 
+    // Mailbox management
+    router.get ('/api/v1/mailboxes', usersScope, routes.user.requireAdmin, routes.mailboxes.list);
+    router.post('/api/v1/mailboxes', usersScope, routes.user.requireAdmin, routes.mailboxes.create);
+    router.get ('/api/v1/mailboxes/:mailboxId', usersScope, routes.user.requireAdmin, routes.mailboxes.get);
+    router.del ('/api/v1/mailboxes/:mailboxId', usersScope, routes.user.requireAdmin, routes.mailboxes.remove);
+    router.put ('/api/v1/mailboxes/:mailboxId/aliases', usersScope, routes.user.requireAdmin, routes.mailboxes.setAliases);
+    router.get ('/api/v1/mailboxes/:mailboxId/aliases', usersScope, routes.user.requireAdmin, routes.mailboxes.getAliases);
+
     // form based login routes used by oauth2 frame
     router.get ('/api/v1/session/login', csrf, routes.oauth2.loginForm);
     router.post('/api/v1/session/login', csrf, routes.oauth2.login);
@@ -136,13 +146,15 @@ function initializeExpressSync() {
     // oauth2 routes
     router.get ('/api/v1/oauth/dialog/authorize', routes.oauth2.authorization);
     router.post('/api/v1/oauth/token', routes.oauth2.token);
-    router.get ('/api/v1/oauth/clients', settingsScope, routes.clients.getAllByUserId);
+    router.get ('/api/v1/oauth/clients', settingsScope, routes.clients.getAll);
     router.post('/api/v1/oauth/clients', routes.developer.enabled, settingsScope, routes.clients.add);
     router.get ('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.get);
     router.post('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.add);
     router.del ('/api/v1/oauth/clients/:clientId', routes.developer.enabled, settingsScope, routes.clients.del);
     router.get ('/api/v1/oauth/clients/:clientId/tokens', settingsScope, routes.clients.getClientTokens);
+    router.post('/api/v1/oauth/clients/:clientId/tokens', routes.developer.enabled, settingsScope, routes.clients.addClientToken);
     router.del ('/api/v1/oauth/clients/:clientId/tokens', settingsScope, routes.clients.delClientTokens);
+    router.del ('/api/v1/oauth/clients/:clientId/tokens/:tokenId', settingsScope, routes.clients.delToken);
 
     // app routes
     router.get ('/api/v1/apps',          appsScope, routes.apps.getApps);
@@ -161,6 +173,7 @@ function initializeExpressSync() {
     router.get ('/api/v1/apps/:id/logstream', appsScope, routes.user.requireAdmin, routes.apps.getLogStream);
     router.get ('/api/v1/apps/:id/logs',      appsScope, routes.user.requireAdmin, routes.apps.getLogs);
     router.get ('/api/v1/apps/:id/exec',      routes.developer.enabled, appsScope, routes.user.requireAdmin, routes.apps.exec);
+    router.post('/api/v1/apps/:id/clone',     appsScope, routes.user.requireAdmin, routes.apps.cloneApp);
 
     // settings routes (these are for the settings tab - avatar & name have public routes for normal users. see above)
     router.get ('/api/v1/settings/autoupdate_pattern', settingsScope, routes.user.requireAdmin, routes.settings.getAutoupdatePattern);
@@ -176,6 +189,7 @@ function initializeExpressSync() {
     router.post('/api/v1/settings/certificate',        settingsScope, routes.user.requireAdmin, routes.settings.setCertificate);
     router.post('/api/v1/settings/admin_certificate',  settingsScope, routes.user.requireAdmin, routes.settings.setAdminCertificate);
     router.get ('/api/v1/settings/time_zone',          settingsScope, routes.user.requireAdmin, routes.settings.getTimeZone);
+    router.post('/api/v1/settings/time_zone',          settingsScope, routes.user.requireAdmin, routes.settings.setTimeZone);
 
     // eventlog route
     router.get('/api/v1/eventlog', settingsScope, routes.user.requireAdmin, routes.eventlog.get);
@@ -183,9 +197,10 @@ function initializeExpressSync() {
     // backup routes
     router.get ('/api/v1/backups', settingsScope, routes.user.requireAdmin, routes.backups.get);
     router.post('/api/v1/backups', settingsScope, routes.user.requireAdmin, routes.backups.create);
-    router.get ('/api/v1/backups/:backupId', appsScope, routes.user.requireAdmin, routes.backups.download);
+    router.post('/api/v1/backups/:backupId/download_url', appsScope, routes.user.requireAdmin, routes.backups.createDownloadUrl);
 
-    // disable server timeout. we use the timeout middleware to handle timeouts on a route level
+    // disable server socket "idle" timeout. we use the timeout middleware to handle timeouts on a route level
+    // we rely on nginx for timeouts on the TCP level (see client_header_timeout)
     httpServer.setTimeout(0);
 
     // upgrade handler
@@ -219,7 +234,7 @@ function initializeSysadminExpressSync() {
     var app = express();
     var httpServer = http.createServer(app);
 
-    var QUERY_LIMIT = '10mb'; // max size for json and urlencoded queries
+    var QUERY_LIMIT = '1mb'; // max size for json and urlencoded queries
     var REQUEST_TIMEOUT = 10000; // timeout for all requests
 
     var json = middleware.json({ strict: true, limit: QUERY_LIMIT }), // application/json
@@ -257,7 +272,7 @@ function start(callback) {
         database.initialize,
         cloudron.initialize, // keep this here because it reads activation state that others depend on
         certificates.installAdminCertificate, // keep this before cron to block heartbeats until cert is ready
-        addons.initialize, // starts the addons
+        platform.initialize,
         taskmanager.initialize,
         mailer.initialize,
         cron.initialize,

src/settings.js

@@ -51,6 +51,7 @@ var assert = require('assert'),
     config = require('./config.js'),
     CronJob = require('cron').CronJob,
     DatabaseError = require('./databaseerror.js'),
+    moment = require('moment-timezone'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     settingsdb = require('./settingsdb.js'),
@@ -62,7 +63,7 @@ var gDefaults = (function () {
     result[exports.AUTOUPDATE_PATTERN_KEY] = '00 00 1,3,5,23 * * *';
     result[exports.TIME_ZONE_KEY] = 'America/Los_Angeles';
     result[exports.CLOUDRON_NAME_KEY] = 'Cloudron';
-    result[exports.DEVELOPER_MODE_KEY] = false;
+    result[exports.DEVELOPER_MODE_KEY] = true;
     result[exports.DNS_CONFIG_KEY] = { };
     result[exports.BACKUP_CONFIG_KEY] = { };
     result[exports.TLS_CONFIG_KEY] = { provider: 'caas' };
@@ -132,6 +133,8 @@ function setTimeZone(tz, callback) {
     assert.strictEqual(typeof tz, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    if (moment.tz.names().indexOf(tz) === -1) return callback(new SettingsError(SettingsError.BAD_FIELD, 'Bad timeZone'));
+
     settingsdb.set(exports.TIME_ZONE_KEY, tz, function (error) {
         if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));
 
@@ -166,7 +169,10 @@ function setCloudronName(name, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    if (!name) return callback(new SettingsError(SettingsError.BAD_FIELD));
+    if (!name) return callback(new SettingsError(SettingsError.BAD_FIELD, 'name is empty'));
+
+    // some arbitrary restrictions (for sake of ui layout)
+    if (name.length > 32) return callback(new SettingsError(SettingsError.BAD_FIELD, 'name cannot exceed 32 characters'));
 
     settingsdb.set(exports.CLOUDRON_NAME_KEY, name, function (error) {
         if (error) return callback(new SettingsError(SettingsError.INTERNAL_ERROR, error));

src/shell.js

@@ -2,7 +2,8 @@
 
 exports = module.exports = {
     sudo: sudo,
-    exec: exec
+    exec: exec,
+    execSync: execSync
 };
 
 var assert = require('assert'),
@@ -13,6 +14,20 @@ var assert = require('assert'),
 
 var SUDO = '/usr/bin/sudo';
 
+function execSync(tag, cmd, callback) {
+    assert.strictEqual(typeof tag, 'string');
+    assert.strictEqual(typeof cmd, 'string');
+
+    debug(cmd);
+    try {
+        child_process.execSync(cmd, { stdio: 'inherit' });
+    } catch (e) {
+        if (callback) return callback(e);
+        throw e;
+    }
+    if (callback) callback();
+}
+
 function exec(tag, file, args, callback) {
     assert.strictEqual(typeof tag, 'string');
     assert.strictEqual(typeof file, 'string');
@@ -35,7 +50,7 @@ function exec(tag, file, args, callback) {
     cp.on('exit', function (code, signal) {
         if (code || signal) debug(tag + ' code: %s, signal: %s', code, signal);
 
-        callback(code === 0 ? null : new Error(util.format('Exited with error %s signal %s', code, signal)));
+        callback(code === 0 ? null : new Error(util.format(tag + ' exited with error %s signal %s', code, signal)));
     });
 
     cp.on('error', function (error) {

src/simpleauth.js

@@ -8,7 +8,6 @@ exports = module.exports = {
 var apps = require('./apps.js'),
     AppsError = apps.AppsError,
     assert = require('assert'),
-    clientdb = require('./clientdb.js'),
     clients = require('./clients.js'),
     ClientsError = clients.ClientsError,
     config = require('./config.js'),
@@ -38,7 +37,7 @@ function loginLogic(clientId, username, password, callback) {
         if (error) return callback(error);
 
         // only allow simple auth clients
-        if (clientObject.type !== clientdb.TYPE_SIMPLE_AUTH) return callback(new ClientsError(ClientsError.INVALID_CLIENT));
+        if (clientObject.type !== clients.TYPE_SIMPLE_AUTH) return callback(new ClientsError(ClientsError.INVALID_CLIENT));
 
         var authFunction = (username.indexOf('@') === -1) ? user.verifyWithUsername : user.verifyWithEmail;
         authFunction(username, password, function (error, userObject) {
@@ -54,7 +53,7 @@ function loginLogic(clientId, username, password, callback) {
                     var accessToken = tokendb.generateToken();
                     var expires = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-                    tokendb.add(accessToken, tokendb.PREFIX_USER + userObject.id, clientId, expires, clientObject.scope, function (error) {
+                    tokendb.add(accessToken, userObject.id, clientId, expires, clientObject.scope, function (error) {
                         if (error) return callback(error);
 
                         debug('login: new access token for client %s and user %s: %s', clientId, username, accessToken);
@@ -87,7 +86,7 @@ function login(req, res, next) {
     if (typeof req.body.password !== 'string') return next(new HttpError(400, 'password is required'));
 
     loginLogic(req.body.clientId, req.body.username, req.body.password, function (error, result) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return next(new HttpError(401, 'Unknown client'));
+        if (error && error.reason === ClientsError.NOT_FOUND) return next(new HttpError(401, 'Unknown client'));
         if (error && error.reason === ClientsError.INVALID_CLIENT) return next(new HttpError(401, 'Unkown client'));
         if (error && error.reason === UserError.NOT_FOUND) return next(new HttpError(401, 'Forbidden'));
         if (error && error.reason === AppsError.NOT_FOUND) return next(new HttpError(401, 'Unkown app'));

src/subdomains.js

@@ -43,7 +43,7 @@ SubdomainError.NOT_FOUND = 'No such domain';
 SubdomainError.EXTERNAL_ERROR = 'External error';
 SubdomainError.STILL_BUSY = 'Still busy';
 SubdomainError.MISSING_CREDENTIALS = 'Missing credentials';
-SubdomainError.INTERNAL_ERROR = 'Missing credentials';
+SubdomainError.INTERNAL_ERROR = 'Internal error';
 SubdomainError.ACCESS_DENIED = 'Access denied';
 
 // choose which subdomain backend we use for test purpose we use route53

src/taskmanager.js

@@ -164,7 +164,7 @@ function stopAppTask(appId, callback) {
     if (gActiveTasks[appId]) {
         debug('stopAppTask : Killing existing task of %s with pid %s', appId, gActiveTasks[appId].pid);
         gActiveTasks[appId].once('exit', function () { callback(); });
-        gActiveTasks[appId].kill('SIGKILL'); // this will end up calling the 'exit' handler
+        gActiveTasks[appId].kill('SIGTERM'); // this will end up calling the 'exit' handler
         return;
     }
 

src/test/apps-test.js

@@ -116,9 +116,9 @@ describe('Apps', function () {
             groups.create.bind(null, GROUP_1),
             groups.addMember.bind(null, groups.ADMIN_GROUP_ID, ADMIN_0.id),
             groups.addMember.bind(null, GROUP_0, USER_1.id),
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, null),
-            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1.accessRestriction, APP_1.memoryLimit, null),
-            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2.accessRestriction, APP_2.memoryLimit, null)
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
+            appdb.add.bind(null, APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, APP_1.portBindings, APP_1),
+            appdb.add.bind(null, APP_2.id, APP_2.appStoreId, APP_2.manifest, APP_2.location, APP_2.portBindings, APP_2)
         ], done);
     });
 
@@ -326,4 +326,57 @@ describe('Apps', function () {
             });
         });
     });
+
+    describe('configureInstalledApps', function () {
+        before(function (done) {
+            async.series([
+                appdb.update.bind(null, APP_0.id, { installationState: appdb.ISTATE_INSTALLED }),
+                appdb.update.bind(null, APP_1.id, { installationState: appdb.ISTATE_ERROR }),
+                appdb.update.bind(null, APP_2.id, { installationState: appdb.ISTATE_INSTALLED })
+            ], done);
+        });
+
+        it('can mark apps for reconfigure', function (done) {
+            apps.configureInstalledApps(function (error) {
+                expect(error).to.be(null);
+
+                apps.getAll(function (error, apps) {
+                    expect(apps[0].installationState).to.be(appdb.ISTATE_PENDING_CONFIGURE);
+                    expect(apps[0].oldConfig).to.be(null);
+                    expect(apps[1].installationState).to.be(appdb.ISTATE_ERROR);
+                    expect(apps[2].installationState).to.be(appdb.ISTATE_PENDING_CONFIGURE);
+                    expect(apps[2].oldConfig).to.be(null);
+
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('restoreInstalledApps', function () {
+        before(function (done) {
+            async.series([
+                appdb.update.bind(null, APP_0.id, { installationState: appdb.ISTATE_INSTALLED }),
+                appdb.update.bind(null, APP_1.id, { installationState: appdb.ISTATE_ERROR }),
+                appdb.update.bind(null, APP_2.id, { installationState: appdb.ISTATE_INSTALLED })
+            ], done);
+        });
+
+        it('can mark apps for reconfigure', function (done) {
+            apps.restoreInstalledApps(function (error) {
+                expect(error).to.be(null);
+
+                apps.getAll(function (error, apps) {
+                    expect(apps[0].installationState).to.be(appdb.ISTATE_PENDING_RESTORE);
+                    expect(apps[0].oldConfig).to.be(null);
+                    expect(apps[1].installationState).to.be(appdb.ISTATE_PENDING_RESTORE);
+                    expect(apps[2].installationState).to.be(appdb.ISTATE_PENDING_RESTORE);
+                    expect(apps[2].oldConfig).to.be(null);
+
+                    done();
+                });
+            });
+        });
+    });
 });
+

src/test/apptask-test.js

@@ -31,7 +31,7 @@ var MANIFEST = {
   "contactEmail": "support@cloudron.io",
   "version": "0.1.0",
   "manifestVersion": 1,
-  "dockerImage": "cloudron/test:8.0.0",
+  "dockerImage": "cloudron/test:16.0.0",
   "healthCheckPath": "/",
   "httpPort": 7777,
   "tcpPorts": {
@@ -84,7 +84,7 @@ describe('apptask', function () {
         config.set('version', '0.5.0');
         async.series([
             database.initialize,
-            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP.accessRestriction, APP.memoryLimit, null),
+            appdb.add.bind(null, APP.id, APP.appStoreId, APP.manifest, APP.location, APP.portBindings, APP),
             settings.setDnsConfig.bind(null, { provider: 'route53', accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', endpoint: 'http://localhost:5353' }),
             settings.setTlsConfig.bind(null, { provider: 'caas' })
         ], done);

src/test/checkInstall

@@ -3,9 +3,7 @@
 set -eu
 
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-readonly TEST_IMAGE="cloudron/test:11.0.0"
-
-source ${SOURCE_DIR}/src/INFRA_VERSION
+readonly TEST_IMAGE="cloudron/test:16.0.0"
 
 # reset sudo timestamp to avoid wrong success
 sudo -k || sudo --reset-timestamp
@@ -13,7 +11,6 @@ sudo -k || sudo --reset-timestamp
 # checks if all scripts are sudo access
 scripts=("${SOURCE_DIR}/src/scripts/rmappdir.sh" \
          "${SOURCE_DIR}/src/scripts/createappdir.sh" \
-         "${SOURCE_DIR}/src/scripts/setup_infra.sh" \
          "${SOURCE_DIR}/src/scripts/reloadnginx.sh" \
          "${SOURCE_DIR}/src/scripts/backupbox.sh" \
          "${SOURCE_DIR}/src/scripts/backupapp.sh" \
@@ -37,35 +34,14 @@ done
 
 image_missing=""
 
-if ! docker inspect "${TEST_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${TEST_IMAGE}"
-    image_missing="true"
-fi
+images=$(node -e "var i = require('${SOURCE_DIR}/src/infra_version.js'); console.log(Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join('\n'));"; echo $TEST_IMAGE)
 
-if ! docker inspect "${REDIS_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${REDIS_IMAGE}"
-    image_missing="true"
-fi
-
-if ! docker inspect "${MYSQL_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${MYSQL_IMAGE}"
-    image_missing="true"
-fi
-
-if ! docker inspect "${POSTGRESQL_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${POSTGRESQL_IMAGE}"
-    image_missing="true"
-fi
-
-if ! docker inspect "${MONGODB_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${MONGODB_IMAGE}"
-    image_missing="true"
-fi
-
-if ! docker inspect "${MAIL_IMAGE}" >/dev/null 2>/dev/null; then
-    echo "docker pull ${MAIL_IMAGE}"
-    image_missing="true"
-fi
+for image in ${images}; do
+    if ! docker inspect "${image}" >/dev/null 2>/dev/null; then
+        echo "docker pull ${image}"
+        image_missing="true"
+    fi
+done
 
 if [[ "${image_missing}" == "true" ]]; then
     echo "Pull above images before running tests"

src/test/database-test.js

@@ -16,6 +16,7 @@ var appdb = require('../appdb.js'),
     eventlogdb = require('../eventlogdb.js'),
     expect = require('expect.js'),
     hat = require('hat'),
+    mailboxdb = require('../mailboxdb.js'),
     settingsdb = require('../settingsdb.js'),
     tokendb = require('../tokendb.js'),
     userdb = require('../userdb.js'),
@@ -85,10 +86,28 @@ describe('database', function () {
             userdb.add(USER_2.id, USER_2, done);
         });
 
-        it('cannot add same user again', function (done) {
-            userdb.add(USER_0.id, USER_0, function (error) {
+        it('cannot add user width same email again', function (done) {
+            var tmp = JSON.parse(JSON.stringify(USER_0));
+            tmp.id = 'somethingelse';
+            tmp.username = 'somethingelse';
+
+            userdb.add(tmp.id, tmp, function (error) {
                 expect(error).to.be.ok();
                 expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
+                expect(error.message).to.equal('email already exists');
+                done();
+            });
+        });
+
+        it('cannot add user width same username again', function (done) {
+            var tmp = JSON.parse(JSON.stringify(USER_0));
+            tmp.id = 'somethingelse';
+            tmp.email = 'somethingelse@not.taken';
+
+            userdb.add(tmp.id, tmp, function (error) {
+                expect(error).to.be.ok();
+                expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
+                expect(error.message).to.equal('username already exists');
                 done();
             });
         });
@@ -184,6 +203,24 @@ describe('database', function () {
             });
         });
 
+        it('can update the user with already existing email', function (done) {
+            userdb.update(USER_0.id, { email: USER_2.email }, function (error) {
+                expect(error).to.be.ok();
+                expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
+                expect(error.message).to.equal('email already exists');
+                done();
+            });
+        });
+
+        it('can update the user with already existing username', function (done) {
+            userdb.update(USER_0.id, { username: USER_2.username }, function (error) {
+                expect(error).to.be.ok();
+                expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
+                expect(error.message).to.equal('username already exists');
+                done();
+            });
+        });
+
         it('cannot update with null field', function () {
             expect(function () {
                 userdb.update(USER_0.id, { email: null }, function () {});
@@ -329,21 +366,21 @@ describe('database', function () {
     describe('token', function () {
         var TOKEN_0 = {
             accessToken: tokendb.generateToken(),
-            identifier: tokendb.PREFIX_USER + '0',
+            identifier: '0',
             clientId: 'clientid-0',
             expires: Date.now() + 60 * 60000,
             scope: '*'
         };
         var TOKEN_1 = {
             accessToken: tokendb.generateToken(),
-            identifier: tokendb.PREFIX_USER + '1',
+            identifier: '1',
             clientId: 'clientid-1',
             expires: Number.MAX_SAFE_INTEGER,
             scope: '*'
         };
         var TOKEN_2 = {
             accessToken: tokendb.generateToken(),
-            identifier: tokendb.PREFIX_USER + '2',
+            identifier: '2',
             clientId: 'clientid-2',
             expires: Date.now(),
             scope: '*'
@@ -470,16 +507,30 @@ describe('database', function () {
         it('delByIdentifierAndClientId succeeds', function (done) {
             tokendb.delByIdentifierAndClientId(TOKEN_0.identifier, TOKEN_0.clientId, function (error) {
                 expect(error).to.be(null);
-                done();
+
+                tokendb.get(TOKEN_0.accessToken, function (error, result) {
+                    expect(error).to.be.a(DatabaseError);
+                    expect(error.reason).to.be(DatabaseError.NOT_FOUND);
+                    expect(result).to.not.be.ok();
+                    done();
+                });
             });
         });
 
-        it('get of previously deleted token fails', function (done) {
-            tokendb.get(TOKEN_0.accessToken, function (error, result) {
-                expect(error).to.be.a(DatabaseError);
-                expect(error.reason).to.be(DatabaseError.NOT_FOUND);
-                expect(result).to.not.be.ok();
-                done();
+        it('delByClientId succeeds', function (done) {
+            tokendb.add(TOKEN_0.accessToken, TOKEN_0.identifier, TOKEN_0.clientId, TOKEN_0.expires, TOKEN_0.scope, function (error) {
+                expect(error).to.be(null);
+
+                tokendb.delByClientId(TOKEN_0.clientId, function (error, result) {
+                    expect(error).to.not.be.ok();
+
+                    tokendb.get(TOKEN_0.accessToken, function (error, result) {
+                        expect(error).to.be.a(DatabaseError);
+                        expect(error.reason).to.be(DatabaseError.NOT_FOUND);
+                        expect(result).to.not.be.ok();
+                        done();
+                    });
+                });
             });
         });
     });
@@ -500,7 +551,6 @@ describe('database', function () {
             health: null,
             accessRestriction: null,
             lastBackupId: null,
-            lastBackupConfig: null,
             oldConfig: null,
             memoryLimit: 4294967296,
             altDomain: null
@@ -520,7 +570,6 @@ describe('database', function () {
             health: null,
             accessRestriction: { users: [ 'foobar' ] },
             lastBackupId: null,
-            lastBackupConfig: null,
             oldConfig: null,
             memoryLimit: 0,
             altDomain: null
@@ -540,7 +589,7 @@ describe('database', function () {
         });
 
         it('add succeeds', function (done) {
-            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, null, function (error) {
+            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0, function (error) {
                 expect(error).to.be(null);
                 done();
             });
@@ -564,7 +613,7 @@ describe('database', function () {
         });
 
         it('add of same app fails', function (done) {
-            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, [ ], APP_0.accessRestriction, APP_0.memoryLimit, null, function (error) {
+            appdb.add(APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, [ ], APP_0, function (error) {
                 expect(error).to.be.a(DatabaseError);
                 expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
                 done();
@@ -636,7 +685,7 @@ describe('database', function () {
         });
 
         it('add second app succeeds', function (done) {
-            appdb.add(APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, [ ], APP_1.accessRestriction, APP_1.memoryLimit, null, function (error) {
+            appdb.add(APP_1.id, APP_1.appStoreId, APP_1.manifest, APP_1.location, [ ], APP_1, function (error) {
                 expect(error).to.be(null);
                 done();
             });
@@ -812,7 +861,7 @@ describe('database', function () {
         var CLIENT_0 = {
             id: 'cid-0',
             appId: 'someappid_0',
-            type: clientdb.TYPE_OAUTH,
+            type: 'typeisastring',
             clientSecret: 'secret-0',
             redirectURI: 'http://foo.bar',
             scope: '*'
@@ -821,7 +870,7 @@ describe('database', function () {
         var CLIENT_1 = {
             id: 'cid-1',
             appId: 'someappid_1',
-            type: clientdb.TYPE_OAUTH,
+            type: 'typeisastring',
             clientSecret: 'secret-',
             redirectURI: 'http://foo.bar',
             scope: '*'
@@ -883,9 +932,9 @@ describe('database', function () {
             clientdb.getAll(function (error, result) {
                 expect(error).to.be(null);
                 expect(result).to.be.an(Array);
-                expect(result.length).to.equal(3); // one of them is webadmin
-                expect(result[0]).to.eql(CLIENT_0);
-                expect(result[1]).to.eql(CLIENT_1);
+                expect(result.length).to.equal(5); // three built-in clients
+                expect(result[3]).to.eql(CLIENT_0);
+                expect(result[4]).to.eql(CLIENT_1);
                 done();
             });
         });
@@ -1123,5 +1172,78 @@ describe('database', function () {
             });
         });
     });
+
+    describe('mailboxes', function () {
+        it('add succeeds', function (done) {
+            mailboxdb.add('support', function (error, mailbox) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+
+        it('cannot add dup entry', function (done) {
+            mailboxdb.add('support', function (error, mailbox) {
+                expect(error.reason).to.be(DatabaseError.ALREADY_EXISTS);
+                done();
+            });
+        });
+
+        it('get succeeds', function (done) {
+            mailboxdb.get('support', function (error, mailbox) {
+                expect(error).to.be(null);
+                expect(mailbox.name).to.be('support');
+                expect(mailbox.creationTime).to.be.a(Date);
+
+                done();
+            });
+        });
+
+        it('getAll succeeds', function (done) {
+            mailboxdb.getAll(function (error, results) {
+                expect(error).to.be(null);
+                expect(results).to.be.an(Array);
+                expect(results.length).to.be(1);
+                expect(results[0].name).to.be('support');
+
+                done();
+            });
+        });
+
+        it('can set alias', function (done) {
+            mailboxdb.setAliases('support2', [ 'support2', 'help' ], function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+
+        it('can get alias', function (done) {
+            mailboxdb.getAliases('support2', function (error, results) {
+                expect(error).to.be(null);
+                expect(results.length).to.be(2);
+                expect(results[0]).to.be('help');
+                expect(results[1]).to.be('support2')
+                done();
+            });
+        });
+
+        it('unset aliases', function (done) {
+            mailboxdb.setAliases('support2', [ ], function (error) {
+                expect(error).to.be(null);
+
+                mailboxdb.getAliases('support2', function (error, results) {
+                    expect(error).to.be(null);
+                    expect(results.length).to.be(0);
+                    done();
+                });
+            });
+        });
+
+        it('del succeeds', function (done) {
+            mailboxdb.del('support', function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+    });
 });
 

src/test/groups-test.js

@@ -53,28 +53,28 @@ describe('Groups', function () {
 
     it('cannot create group - too small', function (done) {
         groups.create('a', function (error) {
-            expect(error.reason).to.be(GroupError.BAD_NAME);
+            expect(error.reason).to.be(GroupError.BAD_FIELD);
             done();
         });
     });
 
     it('cannot create group - too big', function (done) {
         groups.create(new Array(256).join('a'), function (error) {
-            expect(error.reason).to.be(GroupError.BAD_NAME);
+            expect(error.reason).to.be(GroupError.BAD_FIELD);
             done();
         });
     });
 
     it('cannot create group - bad name', function (done) {
         groups.create('bad:name', function (error) {
-            expect(error.reason).to.be(GroupError.BAD_NAME);
+            expect(error.reason).to.be(GroupError.BAD_FIELD);
             done();
         });
     });
 
     it('cannot create group - reserved', function (done) {
         groups.create('users', function (error) {
-            expect(error.reason).to.be(GroupError.BAD_NAME);
+            expect(error.reason).to.be(GroupError.BAD_FIELD);
             done();
         });
     });

src/test/janitor-test.js

@@ -30,14 +30,14 @@ describe('janitor', function () {
 
     var TOKEN_0 = {
         accessToken: tokendb.generateToken(),
-        identifier: tokendb.PREFIX_USER + '0',
+        identifier: '0',
         clientId: 'clientid-0',
         expires: Date.now() + 60 * 60 * 1000,
         scope: '*'
     };
     var TOKEN_1 = {
         accessToken: tokendb.generateToken(),
-        identifier: tokendb.PREFIX_USER + '1',
+        identifier: '1',
         clientId: 'clientid-1',
         expires: Date.now() - 1000,
         scope: '*',

src/test/ldap-test.js

@@ -53,7 +53,6 @@ var APP_0 = {
     health: null,
     accessRestriction: null,
     lastBackupId: null,
-    lastBackupConfig: null,
     oldConfig: null,
     memoryLimit: 4294967296
 };
@@ -72,7 +71,7 @@ function setup(done) {
         database.initialize.bind(null),
         database._clear.bind(null),
         ldapServer.start.bind(null),
-        appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, null),
+        appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
         appdb.update.bind(null, APP_0.id, { containerId: APP_0.containerId }),
         function (callback) {
             user.createOwner(USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE, function (error, result) {
@@ -103,14 +102,14 @@ function setup(done) {
                 answer = [{
                     Name: "irrelevant"
                 }, {
-                    Name: "bridge",
+                    Name: "cloudron",
                     Id: "f2de39df4171b0dc801e8002d1d999b77256983dfc63041c0f34030aa3977566",
                     Scope: "local",
                     Driver: "bridge",
                     IPAM: {
                         Driver: "default",
                         Config: [{
-                            Subnet: "172.17.0.0/16"
+                            Subnet: "172.18.0.0/16"
                         }]
                     },
                     "Containers": {
@@ -139,8 +138,11 @@ function setup(done) {
 }
 
 function cleanup(done) {
-    dockerProxy.close(function () {
-        database._clear(done);
+    async.series([
+        ldapServer.stop,
+        database._clear
+    ], function () {
+        dockerProxy.close(function () { done(); }); // some strange error
     });
 }
 

src/test/mailboxes-test.js

@@ -0,0 +1,136 @@
+/* global it:false */
+/* global describe:false */
+/* global before:false */
+/* global after:false */
+
+'use strict';
+
+var database = require('../database.js'),
+    expect = require('expect.js'),
+    mailboxes = require('../mailboxes.js'),
+    MailboxError = mailboxes.MailboxError,
+    hat = require('hat');
+
+function setup(done) {
+    // ensure data/config/mount paths
+    database.initialize(function (error) {
+        expect(error).to.be(null);
+
+        database._clear(done);
+    });
+}
+
+function cleanup(done) {
+    database._clear(done);
+}
+
+var MAILBOX_NAME = 'test';
+
+describe('Mailboxes', function () {
+    before(setup);
+    after(cleanup);
+
+    it('cannot create mailbox - too small', function (done) {
+        mailboxes.add('a', function (error) {
+            expect(error.reason).to.be(MailboxError.BAD_FIELD);
+            done();
+        });
+    });
+
+    it('cannot create mailbox - too big', function (done) {
+        mailboxes.add(new Array(129).join('a'), function (error) {
+            expect(error.reason).to.be(MailboxError.BAD_FIELD);
+            done();
+        });
+    });
+
+    it('cannot create mailbox - bad name', function (done) {
+        mailboxes.add('bad:name', function (error) {
+            expect(error.reason).to.be(MailboxError.BAD_FIELD);
+            done();
+        });
+    });
+
+    it('cannot create mailbox - reserved', function (done) {
+        mailboxes.add('no-reply', function (error) {
+            expect(error.reason).to.be(MailboxError.BAD_FIELD);
+            done();
+        });
+    });
+
+    it('can create valid mailbox', function (done) {
+        mailboxes.add(MAILBOX_NAME, function (error) {
+            expect(error).to.be(null);
+            done();
+        });
+    });
+
+    it('cannot add existing mailbox', function (done) {
+        mailboxes.add(MAILBOX_NAME, function (error) {
+            expect(error.reason).to.be(MailboxError.ALREADY_EXISTS);
+            done();
+        });
+    });
+
+    it('cannot get invalid mailbox', function (done) {
+        mailboxes.get('sometrandom', function (error) {
+            expect(error.reason).to.be(MailboxError.NOT_FOUND);
+            done();
+        });
+    });
+
+    it('can get valid mailbox', function (done) {
+        mailboxes.get(MAILBOX_NAME, function (error, group) {
+            expect(error).to.be(null);
+            expect(group.name).to.equal(MAILBOX_NAME);
+            done();
+        });
+    });
+
+    it('can set aliases', function (done) {
+        mailboxes.setAliases(MAILBOX_NAME, [ 'alias1', 'alias2' ], function (error) {
+            expect(error).to.be(null);
+            done();
+        });
+    });
+
+    it('can set subset alias', function (done) {
+        mailboxes.setAliases(MAILBOX_NAME, [ 'alias1' ], function (error) {
+            expect(error).to.be(null);
+            done();
+        });
+    });
+
+    it('can get aliases', function (done) {
+        mailboxes.getAliases(MAILBOX_NAME, function (error, aliases) {
+            expect(error).to.be(null);
+            expect(aliases[0]).to.be('alias1');
+            done();
+        });
+    });
+
+    it('can get aliases from mailbox', function (done) {
+        mailboxes.get(MAILBOX_NAME, function (error, group) {
+            expect(error).to.be(null);
+            expect(group.name).to.equal(MAILBOX_NAME);
+            expect(group.aliases.length).to.be(1);
+            expect(group.aliases[0]).to.be('alias1');
+            done();
+        });
+    });
+
+    it('cannot set self-referential alias', function (done) {
+        mailboxes.setAliases(MAILBOX_NAME, [ MAILBOX_NAME ], function (error) {
+            expect(error.reason).to.be(MailboxError.ALREADY_EXISTS);
+            done();
+        });
+    });
+
+    it('cannot delete invalid mailbox', function (done) {
+        mailboxes.del('random', function (error) {
+            expect(error.reason).to.be(MailboxError.NOT_FOUND);
+            done();
+        });
+    });
+});
+

src/test/settings-test.js

@@ -1,4 +1,3 @@
-/* jslint node:true */
 /* global it:false */
 /* global describe:false */
 /* global before:false */
@@ -62,7 +61,7 @@ describe('Settings', function () {
         it('can get default developer mode', function (done) {
             settings.getDeveloperMode(function (error, enabled) {
                 expect(error).to.be(null);
-                expect(enabled).to.equal(false);
+                expect(enabled).to.equal(true);
                 done();
             });
         });

src/test/setupTest

@@ -8,13 +8,29 @@ readonly source_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
 ! "${source_dir}/src/test/checkInstall" && exit 1
 
 # create dir structure
-rm -rf $HOME/.cloudron_test
+rm -rf $HOME/.cloudron_test 2>/dev/null || true # some of those docker container data requires sudo to be removed
 mkdir -p $HOME/.cloudron_test
 cd $HOME/.cloudron_test
-mkdir -p data/appdata data/box/appicons data/mail data/nginx/cert data/nginx/applications data/collectd/collectd.conf.d data/addons configs data/box/certs data/box/mail/dkim/localhost
+mkdir -p data/appdata data/box/appicons data/mail data/nginx/cert data/nginx/applications data/collectd/collectd.conf.d data/addons configs data/box/certs data/box/mail/dkim/localhost data/box/mail/dkim/foobar.com
 
-webadmin_scopes="root,profile,users,apps,settings"
+# put cert
+openssl req -x509 -newkey rsa:2048 -keyout data/nginx/cert/host.key -out data/nginx/cert/host.cert -days 3650 -subj '/CN=localhost' -nodes
+
+webadmin_scopes="cloudron,profile,users,apps,settings"
 webadmin_origin="https://${ADMIN_LOCATION}-localhost"
-mysql --user=root --password="" \
-    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"webadmin\", \"admin\", \"secret-webadmin\", \"${webadmin_origin}\", \"${webadmin_scopes}\")" boxtest
 
+# create docker network (while the infra code does this, most tests skip infra setup)
+docker network create cloudron --subnet=172.18.0.0/16 || true
+
+# !!!!!! check clientdb.js clear() to not nuke those entries
+echo "Add webadmin api client"
+mysql --user=root --password="" \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-webadmin\", \"Settings\", \"built-in\", \"secret-webadmin\", \"${webadmin_origin}\", \"${webadmin_scopes}\")" boxtest
+
+echo "Add SDK api client"
+mysql --user=root --password="" \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-sdk\", \"SDK\", \"built-in\", \"secret-sdk\", \"${webadmin_origin}\", \"*,roleSdk\")" boxtest
+
+echo "Add cli api client"
+mysql --user=root --password="" \
+    -e "REPLACE INTO clients (id, appId, type, clientSecret, redirectURI, scope) VALUES (\"cid-cli\", \"Cloudron Tool\", \"built-in\", \"secret-cli\", \"${webadmin_origin}\", \"*,roleSdk\")" boxtest

src/test/shell-test.js

@@ -47,5 +47,20 @@ describe('shell', function () {
             done();
         });
     });
+
+    it('execSync a valid program', function (done) {
+        shell.execSync('test', 'ls -l | wc -c');
+        done();
+    });
+
+    it('execSync throws for invalid program', function (done) {
+        expect(function () { shell.execSync('test', 'cannotexist') }).to.throwException();
+        done();
+    });
+
+    it('execSync throws for failed program', function (done) {
+        expect(function () { shell.execSync('test', 'false'); }).to.throwException();
+        done();
+    });
 });
 

src/test/updatechecker-test.js

@@ -251,7 +251,7 @@ describe('updatechecker - checkAppUpdates', function () {
             database.initialize,
             database._clear,
             mailer._clearMailQueue,
-            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0.accessRestriction, APP_0.memoryLimit, null),
+            appdb.add.bind(null, APP_0.id, APP_0.appStoreId, APP_0.manifest, APP_0.location, APP_0.portBindings, APP_0),
             user.createOwner.bind(null, USER_0.username, USER_0.password, USER_0.email, USER_0.displayName, AUDIT_SOURCE)
         ], done);
     });

src/test/user-test.js

@@ -1,4 +1,3 @@
-/* jslint node:true */
 /* global it:false */
 /* global describe:false */
 /* global before:false */
@@ -86,7 +85,7 @@ describe('User', function () {
             user.create(USERNAME, 'Fo$%23', EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).to.not.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_PASSWORD);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
@@ -96,7 +95,7 @@ describe('User', function () {
             user.create(USERNAME, 'thisiseightch%$234arslong', EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).to.not.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_PASSWORD);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
@@ -106,7 +105,7 @@ describe('User', function () {
             user.create(USERNAME, 'foobaRASDF%', EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).to.not.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_PASSWORD);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
@@ -116,7 +115,7 @@ describe('User', function () {
             user.create(USERNAME, 'foobaRASDF23423', EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).to.not.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_PASSWORD);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
@@ -126,17 +125,47 @@ describe('User', function () {
             user.create('admin', PASSWORD, EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).to.not.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_USERNAME);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
         });
 
         it('fails due to reserved username', function (done) {
-            user.create('AdMiN', PASSWORD, EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
+            user.create('Mailer-Daemon', PASSWORD, EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).to.not.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_USERNAME);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('fails due to short username', function (done) {
+            user.create('Z', PASSWORD, EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
+                expect(error).to.be.ok();
+                expect(result).to.not.be.ok();
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('fails due to long username', function (done) {
+            user.create(new Array(257).fill('Z').join(''), PASSWORD, EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
+                expect(error).to.be.ok();
+                expect(result).to.not.be.ok();
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
+
+                done();
+            });
+        });
+
+        it('fails due to reserved pattern', function (done) {
+            user.create('maybe-app', PASSWORD, EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
+                expect(error).to.be.ok();
+                expect(result).to.not.be.ok();
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
@@ -191,7 +220,7 @@ describe('User', function () {
             user.create(USERNAME, '', EMAIL, DISPLAY_NAME, AUDIT_SOURCE, function (error, result) {
                 expect(error).to.be.ok();
                 expect(result).not.to.be.ok();
-                expect(error.reason).to.equal(UserError.BAD_PASSWORD);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
@@ -404,7 +433,8 @@ describe('User', function () {
         after(cleanupUsers);
 
         it('fails due to unknown userid', function (done) {
-            user.update(USERNAME, USERNAME_NEW, EMAIL_NEW, DISPLAY_NAME_NEW, AUDIT_SOURCE, function (error) {
+            var data = { username: USERNAME_NEW, email: EMAIL_NEW, displayName: DISPLAY_NAME_NEW };
+            user.update(USERNAME, data, AUDIT_SOURCE, function (error) {
                 expect(error).to.be.a(UserError);
                 expect(error.reason).to.equal(UserError.NOT_FOUND);
 
@@ -413,16 +443,19 @@ describe('User', function () {
         });
 
         it('fails due to invalid email', function (done) {
-            user.update(userObject.id, USERNAME_NEW, 'brokenemailaddress', DISPLAY_NAME_NEW, AUDIT_SOURCE, function (error) {
+            var data = { username: USERNAME_NEW, email: 'brokenemailaddress', displayName: DISPLAY_NAME_NEW };
+            user.update(userObject.id, data, AUDIT_SOURCE, function (error) {
                 expect(error).to.be.a(UserError);
-                expect(error.reason).to.equal(UserError.BAD_EMAIL);
+                expect(error.reason).to.equal(UserError.BAD_FIELD);
 
                 done();
             });
         });
 
         it('succeeds', function (done) {
-            user.update(userObject.id, USERNAME_NEW, EMAIL_NEW, DISPLAY_NAME_NEW, AUDIT_SOURCE, function (error) {
+            var data = { username: USERNAME_NEW, email: EMAIL_NEW, displayName: DISPLAY_NAME_NEW };
+
+            user.update(userObject.id, data, AUDIT_SOURCE, function (error) {
                 expect(error).to.not.be.ok();
 
                 user.get(userObject.id, function (error, result) {
@@ -438,7 +471,9 @@ describe('User', function () {
         });
 
         it('succeeds with same data', function (done) {
-            user.update(userObject.id, USERNAME_NEW, EMAIL_NEW, DISPLAY_NAME_NEW, AUDIT_SOURCE, function (error) {
+            var data = { username: USERNAME_NEW, email: EMAIL_NEW, displayName: DISPLAY_NAME_NEW };
+
+            user.update(userObject.id, data, AUDIT_SOURCE, function (error) {
                 expect(error).to.not.be.ok();
 
                 user.get(userObject.id, function (error, result) {
@@ -543,6 +578,19 @@ describe('User', function () {
         });
     });
 
+    describe('count', function () {
+        before(createOwner);
+        after(cleanupUsers);
+
+        it('succeeds', function (done) {
+            user.count(function (error, count) {
+                expect(error).to.not.be.ok();
+                expect(count).to.be(1);
+                done();
+            });
+        });
+    });
+
     describe('set password', function () {
         before(createOwner);
         after(cleanupUsers);
@@ -648,4 +696,23 @@ describe('User', function () {
             });
         });
     });
+
+    describe('remove', function () {
+        before(createOwner);
+        after(cleanupUsers);
+
+        it('fails for unkown user', function (done) {
+            user.remove('unknown', { }, function (error) {
+                expect(error.reason).to.be(UserError.NOT_FOUND);
+                done();
+            });
+        });
+
+        it('can remove valid user', function (done) {
+            user.remove(userObject.id, { }, function (error) {
+                expect(error).to.be(null);
+                done();
+            });
+        });
+    });
 });

src/tokendb.js

@@ -7,20 +7,13 @@ exports = module.exports = {
     get: get,
     add: add,
     del: del,
+    delByClientId: delByClientId,
     getByIdentifier: getByIdentifier,
     delByIdentifier: delByIdentifier,
     getByIdentifierAndClientId: getByIdentifierAndClientId,
     delByIdentifierAndClientId: delByIdentifierAndClientId,
     delExpired: delExpired,
 
-    TYPE_USER: 'user',
-    TYPE_DEV: 'developer',
-    TYPE_APP: 'appliation',
-
-    PREFIX_USER: 'user-',
-    PREFIX_DEV: 'dev-',
-    PREFIX_APP: 'app-',
-
     _clear: clear
 };
 
@@ -33,7 +26,7 @@ var assert = require('assert'),
 var TOKENS_FIELDS = [ 'accessToken', 'identifier', 'clientId', 'scope', 'expires' ].join(',');
 
 function generateToken() {
-    return hat(256);
+    return hat(8 * 64); // TODO: make this stronger
 }
 
 function get(accessToken, callback) {
@@ -77,6 +70,18 @@ function del(accessToken, callback) {
     });
 }
 
+function delByClientId(clientId, callback) {
+    assert.strictEqual(typeof clientId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('DELETE FROM tokens WHERE clientId = ?', [ clientId ], function (error, result) {
+        if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
+        if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
+
+        return callback(null);
+    });
+}
+
 function getByIdentifier(identifier, callback) {
     assert.strictEqual(typeof identifier, 'string');
     assert.strictEqual(typeof callback, 'function');

src/user.js

@@ -5,6 +5,7 @@ exports = module.exports = {
 
     list: listUsers,
     create: createUser,
+    count: count,
     verify: verify,
     verifyWithUsername: verifyWithUsername,
     verifyWithEmail: verifyWithEmail,
@@ -23,14 +24,16 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    clientdb = require('./clientdb.js'),
+    clients = require('./clients.js'),
     crypto = require('crypto'),
+    debug = require('debug')('box:user'),
     DatabaseError = require('./databaseerror.js'),
     eventlog = require('./eventlog.js'),
     groups = require('./groups.js'),
     GroupError = groups.GroupError,
     hat = require('hat'),
     mailer = require('./mailer.js'),
+    mailboxes = require('./mailboxes.js'),
     tokendb = require('./tokendb.js'),
     userdb = require('./userdb.js'),
     util = require('util'),
@@ -43,6 +46,8 @@ var CRYPTO_SALT_SIZE = 64; // 512-bit salt
 var CRYPTO_ITERATIONS = 10000; // iterations
 var CRYPTO_KEY_LENGTH = 512; // bits
 
+var NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 // http://dustinsenos.com/articles/customErrorsInNode
 // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi
 function UserError(reason, errorOrMessage) {
@@ -69,24 +74,27 @@ UserError.ALREADY_EXISTS = 'Already Exists';
 UserError.NOT_FOUND = 'Not Found';
 UserError.WRONG_PASSWORD = 'Wrong User or Password';
 UserError.BAD_FIELD = 'Bad field';
-UserError.BAD_USERNAME = 'Bad username';
-UserError.BAD_EMAIL = 'Bad email';
-UserError.BAD_PASSWORD = 'Bad password';
 UserError.BAD_TOKEN = 'Bad token';
 
 function validateUsername(username) {
     assert.strictEqual(typeof username, 'string');
     // https://github.com/gogits/gogs/blob/52c8f691630548fe091d30bcfe8164545a05d3d5/models/repo.go#L393
     // admin@fqdn is also reservd for sending emails
-    var RESERVED_USERNAMES = [ 'admin', 'no-reply' ]; // apps like wordpress, gogs don't like these
+    var RESERVED_USERNAMES = [ 'admin', 'no-reply', 'postmaster', 'mailer-daemon' ]; // apps like wordpress, gogs don't like these
 
     // allow empty usernames
     if (username === '') return null;
 
-    if (username.length <= 2) return new UserError(UserError.BAD_USERNAME, 'Username must be atleast 3 chars');
-    if (username.length > 256) return new UserError(UserError.BAD_USERNAME, 'Username too long');
+    if (username.length <= 1) return new UserError(UserError.BAD_FIELD, 'Username must be atleast 2 chars');
+    if (username.length > 256) return new UserError(UserError.BAD_FIELD, 'Username too long');
 
-    if (RESERVED_USERNAMES.indexOf(username) !== -1) return new UserError(UserError.BAD_USERNAME, 'Username is reserved');
+    if (RESERVED_USERNAMES.indexOf(username) !== -1) return new UserError(UserError.BAD_FIELD, 'Username is reserved');
+
+    // +/- can be tricky in emails
+    if (/[^a-zA-Z0-9.]/.test(username)) return new UserError(UserError.BAD_FIELD, 'Username can only contain alphanumerals and dot');
+
+    // app emails are sent using the .app suffix
+    if (username.indexOf('.app') !== -1) return new UserError(UserError.BAD_FIELD, 'Username pattern is reserved for apps');
 
     return null;
 }
@@ -94,7 +102,7 @@ function validateUsername(username) {
 function validateEmail(email) {
     assert.strictEqual(typeof email, 'string');
 
-    if (!validator.isEmail(email)) return new UserError(UserError.BAD_EMAIL, 'Invalid email');
+    if (!validator.isEmail(email)) return new UserError(UserError.BAD_FIELD, 'Invalid email');
 
     return null;
 }
@@ -137,7 +145,7 @@ function createUser(username, password, email, displayName, auditSource, options
     if (error) return callback(error);
 
     error = validatePassword(password);
-    if (error) return callback(new UserError(UserError.BAD_PASSWORD, error.message));
+    if (error) return callback(new UserError(UserError.BAD_FIELD, error.message));
 
     error = validateEmail(email);
     if (error) return callback(error);
@@ -166,10 +174,11 @@ function createUser(username, password, email, displayName, auditSource, options
             };
 
             userdb.add(user.id, user, function (error) {
-                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new UserError(UserError.ALREADY_EXISTS));
+                if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new UserError(UserError.ALREADY_EXISTS, error.message));
                 if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
                 eventlog.add(eventlog.ACTION_USER_ADD, auditSource, { userId: user.id, email: user.email });
+                if (username) mailboxes.add(username, NOOP_CALLBACK);
 
                 callback(null, user);
 
@@ -243,35 +252,48 @@ function verifyWithEmail(email, password, callback) {
     });
 }
 
-function removeUser(user, auditSource, callback) {
-    assert.strictEqual(typeof user, 'object');
+function removeUser(userId, auditSource, callback) {
+    assert.strictEqual(typeof userId, 'string');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.del(user.id, function (error) {
-        if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
-        if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
+    getUser(userId, function (error, user) {
+        if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_USER_REMOVE, auditSource, { userId: user.id });
+        userdb.del(userId, function (error) {
+            if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
+            if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
-        callback(null);
+            eventlog.add(eventlog.ACTION_USER_REMOVE, auditSource, { userId: userId });
+            if (user.username) mailboxes.del(user.username, NOOP_CALLBACK);
 
-        mailer.userRemoved(user);
+            callback(null);
+
+            mailer.userRemoved(user);
+        });
     });
 }
 
 function listUsers(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    userdb.getAllWithGroupIds(function (error, result) {
+    userdb.getAllWithGroupIds(function (error, results) {
         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
-        var allUsers = result.map(function (obj) {
-            var u = _.pick(obj, 'id', 'username', 'email', 'displayName', 'groupIds');
-            u.admin = u.groupIds.indexOf(groups.ADMIN_GROUP_ID) !== -1;
-            return u;
+        results.forEach(function (result) {
+            result.admin = result.groupIds.indexOf(groups.ADMIN_GROUP_ID) !== -1;
         });
-        return callback(null, allUsers);
+        return callback(null, results);
+    });
+}
+
+function count(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    userdb.count(function (error, count) {
+        if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
+
+        callback(null, count);
     });
 }
 
@@ -287,6 +309,7 @@ function getUser(userId, callback) {
             if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
             result.groupIds = groupIds;
+            result.admin = groupIds.indexOf(groups.ADMIN_GROUP_ID) !== -1;
 
             return callback(null, result);
         });
@@ -308,29 +331,36 @@ function getByResetToken(resetToken, callback) {
     });
 }
 
-function updateUser(userId, username, email, displayName, auditSource, callback) {
+function updateUser(userId, data, auditSource, callback) {
     assert.strictEqual(typeof userId, 'string');
-    assert.strictEqual(typeof username, 'string');
-    assert.strictEqual(typeof email, 'string');
-    assert.strictEqual(typeof displayName, 'string');
+    assert.strictEqual(typeof data, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    username = username.toLowerCase();
-    email = email.toLowerCase();
+    var error;
+    data = _.pick(data, 'email', 'displayName', 'username');
 
-    var error = validateUsername(username);
-    if (error) return callback(error);
+    if (_.isEmpty(data)) return callback();
 
-    error = validateEmail(email);
-    if (error) return callback(error);
+    if (data.username) {
+        data.username = data.username.toLowerCase();
+        error = validateUsername(data.username);
+        if (error) return callback(error);
+    }
 
-    userdb.update(userId, { username: username, email: email, displayName: displayName }, function (error) {
-        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new UserError(UserError.ALREADY_EXISTS, error));
+    if (data.email) {
+        data.email = data.email.toLowerCase();
+        error = validateEmail(data.email);
+        if (error) return callback(error);
+    }
+
+    userdb.update(userId, data, function (error) {
+        if (error && error.reason === DatabaseError.ALREADY_EXISTS) return callback(new UserError(UserError.ALREADY_EXISTS, error.message));
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND, error));
         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
         eventlog.add(eventlog.ACTION_USER_UPDATE, auditSource, { userId: userId });
+        if (data.username) mailboxes.add(data.username, NOOP_CALLBACK); // TODO: do this only when username actually changes
 
         callback(null);
     });
@@ -406,7 +436,7 @@ function setPassword(userId, newPassword, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var error = validatePassword(newPassword);
-    if (error) return callback(new UserError(UserError.BAD_PASSWORD, error.message));
+    if (error) return callback(new UserError(UserError.BAD_FIELD, error.message));
 
     userdb.get(userId, function (error, user) {
         if (error && error.reason === DatabaseError.NOT_FOUND) return callback(new UserError(UserError.NOT_FOUND));
@@ -425,13 +455,13 @@ function setPassword(userId, newPassword, callback) {
                 if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
                 // Also generate a token so the new user can get logged in immediately
-                clientdb.getByAppIdAndType('webadmin', clientdb.TYPE_ADMIN, function (error, result) {
+                clients.get('cid-webadmin', function (error, result) {
                     if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
                     var token = tokendb.generateToken();
                     var expiresAt = Date.now() + 24 * 60 * 60 * 1000; // 1 day
 
-                    tokendb.add(token, tokendb.PREFIX_USER + user.id, result.id, expiresAt, '*', function (error) {
+                    tokendb.add(token, user.id, result.id, expiresAt, '*', function (error) {
                         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
 
                         callback(null, { token: token, expiresAt: expiresAt });
@@ -451,11 +481,11 @@ function createOwner(username, password, email, displayName, auditSource, callba
     assert.strictEqual(typeof callback, 'function');
 
     // This is only not allowed for the owner
-    if (username === '') return callback(new UserError(UserError.BAD_USERNAME, 'Username cannot be empty'));
+    if (username === '') return callback(new UserError(UserError.BAD_FIELD, 'Username cannot be empty'));
 
     userdb.count(function (error, count) {
         if (error) return callback(new UserError(UserError.INTERNAL_ERROR, error));
-        if (count !== 0) return callback(new UserError(UserError.ALREADY_EXISTS));
+        if (count !== 0) return callback(new UserError(UserError.ALREADY_EXISTS, 'Owner already exists'));
 
         createUser(username, password, email, displayName, auditSource, { owner: true }, function (error, user) {
             if (error) return callback(error);

src/userdb.js

@@ -143,7 +143,16 @@ function add(userId, user, callback) {
 
     var data = [ userId, user.username || null, user.password, user.email, user.salt, user.createdAt, user.modifiedAt, user.resetToken, user.displayName, user.showTutorial ];
     database.query('INSERT INTO users (id, username, password, email, salt, createdAt, modifiedAt, resetToken, displayName, showTutorial) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', data, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_DUP_ENTRY') {
+            var msg = error.message;
+            if (error.message.indexOf('users_email') !== -1) {
+                msg = 'email already exists';
+            } else if (error.message.indexOf('users_username') !== -1) {
+                msg = 'username already exists';
+            }
+
+            return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, msg));
+        }
         if (error || result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
 
         callback(null);
@@ -216,7 +225,16 @@ function update(userId, user, callback) {
     args.push(userId);
 
     database.query('UPDATE users SET ' + fields.join(', ') + ' WHERE id = ?', args, function (error, result) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, error));
+        if (error && error.code === 'ER_DUP_ENTRY') {
+            var msg = error.message;
+            if (error.message.indexOf('users_email') !== -1) {
+                msg = 'email already exists';
+            } else if (error.message.indexOf('users_username') !== -1) {
+                msg = 'username already exists';
+            }
+
+            return callback(new DatabaseError(DatabaseError.ALREADY_EXISTS, msg));
+        }
         if (error) return callback(new DatabaseError(DatabaseError.INTERNAL_ERROR, error));
         if (result.affectedRows !== 1) return callback(new DatabaseError(DatabaseError.NOT_FOUND));
 

webadmin/src/index.html

@@ -4,6 +4,7 @@
     <meta charset="utf-8" />
     <meta name="viewport" content="user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1, width=device-width, height=device-height" />
 
+    <!-- this gets changed once we get the config (because angular has not loaded yet, we see template string for a flash) -->
     <title> Cloudron </title>
 
     <link id="favicon" href="/api/v1/cloudron/avatar" rel="icon" type="image/png">
@@ -173,7 +174,7 @@
                         <span class="icon-bar"></span>
                     </button>
                     <a class="navbar-brand navbar-brand-icon" href="#/"><img ng-src="{{ client.avatar }}" width="40" height="40"/></a>
-                    <a class="navbar-brand" href="#/">Cloudron</a>
+                    <a class="navbar-brand" href="#/">{{ config.cloudronName || 'Cloudron' }}</a>
                 </div>
                 <!-- /.navbar-header -->
 
@@ -201,6 +202,7 @@
                                 <li ng-show="user.admin"><a href="#/activity"><i class="fa fa-list-alt fa-fw"></i> Activity</a></li>
                                 <li><a href="#/support"><i class="fa fa-comment fa-fw"></i> Support</a></li>
                                 <li ng-show="user.admin && config.isCustomDomain"><a href="#/certs"><i class="fa fa-certificate fa-fw"></i> DNS & Certs</a></li>
+                                <li ng-show="user.admin"><a href="#/tokens"><i class="fa fa-key fa-fw"></i> API Access</a></li>
                                 <li ng-show="user.admin"><a href="#/settings"><i class="fa fa-wrench fa-fw"></i> Settings</a></li>
                                 <li class="divider"></li>
                                 <li><a href="" ng-click="logout($event)"><i class="fa fa-sign-out fa-fw"></i> Logout</a></li>
@@ -217,14 +219,14 @@
         </div>
     </div>
 
-    <div class="upgrade hide">
+<!--     <div class="upgrade hide">
         <div class="content">
             <h4>Your Cloudron trial ends soon</h4>
             <p>To keep your Cloudron, just <a href="https://cloudron.io/console.html#/billing" target="_blank">setup a payment method at cloudron.io</a> or <a href="mailto: support@cloudron.io">send us an email</a>.</p>
         </div>
 
         <div class="trigger">Want to keep your Cloudron?</div>
-    </div>
+    </div> -->
 
     <!-- Footer -->
     <footer class="text-center">