5.0.5 changes

CHANGES

@@ -1850,218 +1850,3 @@
 * Increase token expiry
 * Fix bug in tag UI where tag removal did not work
 
-[5.0.6]
-* Make mail eventlog only visible to owners
-* Make app password work with sftp
-
-[5.1.0]
-* Add turn addon
-* Fix disk usage display
-* Drop support for TLSv1 and TLSv1.1
-* Make cert validation work for ECC certs
-* Add type filter to mail eventlog
-* mail: Fix listing of mailboxes and aliases in the UI
-* branding: fix login page title
-* Only a Cloudron owner can install/update/exec apps with the docker addon
-* security: reset tokens are only valid for a day
-* mail: fix eventlog db perms
-* Fix various bugs in the disk graphs
-
-[5.1.1]
-* Add turn addon
-* Fix disk usage display
-* Drop support for TLSv1 and TLSv1.1
-* Make cert validation work for ECC certs
-* Add type filter to mail eventlog
-* mail: Fix listing of mailboxes and aliases in the UI
-* branding: fix login page title
-* Only a Cloudron owner can install/update/exec apps with the docker addon
-* security: reset tokens are only valid for a day
-* mail: fix eventlog db perms
-* Fix various bugs in the disk graphs
-* Fix collectd installation
-* graphs: sort disk contents by usage
-* backups: show apps that are not automatically backed up in backup view
-
-[5.1.2]
-* Add turn addon
-* Fix disk usage display
-* Drop support for TLSv1 and TLSv1.1
-* Make cert validation work for ECC certs
-* Add type filter to mail eventlog
-* mail: Fix listing of mailboxes and aliases in the UI
-* branding: fix login page title
-* Only a Cloudron owner can install/update/exec apps with the docker addon
-* security: reset tokens are only valid for a day
-* mail: fix eventlog db perms
-* Fix various bugs in the disk graphs
-* Fix collectd installation
-* graphs: sort disk contents by usage
-* backups: show apps that are not automatically backed up in backup view
-* turn: deny local address peers https://www.rtcsec.com/2020/04/01-slack-webrtc-turn-compromise/
-
-[5.1.3]
-* Fix crash with misconfigured reverse proxy
-* Fix issue where invitation links are not working anymore
-
-[5.1.4]
-* Add support for custom .well-known documents to be served
-* Add ECDHE-RSA-AES128-SHA256 to cipher list
-* Fix GPG signature verification
-
-[5.1.5]
-* Check for .well-known routes upstream as fallback. This broke nextcloud's caldav/carddav
-
-[5.2.0]
-* acme: request ECC certs
-* less-strict DKIM check to allow users to set a stronger DKIM key
-* Add members only flag to mailing list
-* oauth: add backward compat layer for backup and uninstall
-* fix bug in disk usage sorting
-* mail: aliases can be across domains
-* mail: allow an external MX to be set
-* Add UI to download backup config as JSON (and import it)
-* Ensure stopped apps are getting backed up
-* Add OVH Object Storage backend
-* Add per-app redis status and configuration to Services
-* spam: large emails were not scanned
-* mail relay: fix delivery event log
-* manual update check always gets the latest updates
-* graphs: fix issue where large number of apps would crash the box code (query param limit exceeded)
-* backups: fix various security issues in encypted backups (thanks @mehdi)
-* graphs: add app graphs
-* older encrypted backups cannot be used in this version
-* Add backup listing UI
-* stopping an app will stop dependent services
-* Add new wasabi s3 storage region us-east-2
-* mail: Fix bug where SRS translation was done on the main domain instead of mailing list domain
-* backups: add retention policy
-* Drop `NET_RAW` caps from container preventing sniffing of network traffic
-
-[5.2.1]
-* Fix app disk graphs
-* restart apps on addon container change
-
-[5.2.2]
-* regression: import UI
-* Mbps -> MBps
-* Remove verbose logs
-* Set dmode in tar extract
-* mail: fix crash in audit logs
-* import: fix crash because encryption is unset
-* create redis with the correct label
-
-[5.2.3]
-* Do not restart stopped apps
-
-[5.2.4]
-* mail: enable/disable incoming mail was showing an error
-* Do not trigger backup of stopped apps. Instead, we will just retain it's existing backups
-  based on retention policy
-* remove broken disk graphs
-* fix OVH backups
-
-[5.3.0]
-* better nginx config for higher loads
-* backups: add CIFS storage provider
-* backups: add SSHFS storage provider
-* backups: add NFS storage provider
-* s3: use vhost style
-* Fix crash when redis config was set
-* Update schedule was unselected in the UI
-* cloudron-setup: --provider is now optional
-* show warning for unstable updates
-* add forumUrl to app manifest
-* postgresql: add unaccent extension for peertube
-* mail: Add Auto-Submitted header to NDRs
-* backups: ensure that the latest backup of installed apps is always preserved
-* add nginx logs
-* mail: make authentication case insensitive
-* Fix timeout issues in postgresql and mysql addon
-* Do not count stopped apps for memory use
-* LDAP group synchronization
-
-[5.3.1]
-* better nginx config for higher loads
-* backups: add CIFS storage provider
-* backups: add SSHFS storage provider
-* backups: add NFS storage provider
-* s3: use vhost style
-* Fix crash when redis config was set
-* Update schedule was unselected in the UI
-* cloudron-setup: --provider is now optional
-* show warning for unstable updates
-* add forumUrl to app manifest
-* postgresql: add unaccent extension for peertube
-* mail: Add Auto-Submitted header to NDRs
-* backups: ensure that the latest backup of installed apps is always preserved
-* add nginx logs
-* mail: make authentication case insensitive
-* Fix timeout issues in postgresql and mysql addon
-* Do not count stopped apps for memory use
-* LDAP group synchronization
-
-[5.3.2]
-* Do not install sshfs package
-* 'provider' is not required anymore in various API calls
-* redis: Set maxmemory and maxmemory-policy
-* Add mlock capability to manifest (for vault app)
-
-[5.3.3]
-* Fix issue where some postinstall messages where causing angular to infinite loop
-
-[5.3.4]
-* Fix issue in database error handling
-
-[5.4.0]
-* Update nginx to 1.18 for various security fixes
-* Add ping capability (for statping app)
-* Fix bug where aliases were displayed incorrectly in SOGo
-* Add univention as LDAP provider
-* Bump max_connection for postgres addon to 200
-* mail: Add pagination to mailing list API
-* Allow admin to lock email and display name of users
-* Allow admin to ensure all users have 2FA setup
-* ami: fix regression where we didn't send provider as part of get status call
-* nginx: hide version
-* backups: add b2 provider
-* Add filemanager webinterface
-* Add darkmode
-* Add note that password reset and invite links expire in 24 hours
-
-[5.4.1]
-* Update nginx to 1.18 for various security fixes
-* Add ping capability (for statping app)
-* Fix bug where aliases were displayed incorrectly in SOGo
-* Add univention as LDAP provider
-* Bump max_connection for postgres addon to 200
-* mail: Add pagination to mailing list API
-* Allow admin to lock email and display name of users
-* Allow admin to ensure all users have 2FA setup
-* ami: fix regression where we didn't send provider as part of get status call
-* nginx: hide version
-* backups: add b2 provider
-* Add filemanager webinterface
-* Add darkmode
-* Add note that password reset and invite links expire in 24 hours
-
-[5.5.0]
-* postgresql: update to PostgreSQL 11
-* postgresql: add citext extension to whitelist for loomio
-* postgresql: add btree_gist,postgres_fdw,pg_stat_statements,plpgsql extensions for gitlab
-* SFTP/Filebrowser: fix access of external data directories
-* Fix contrast issues in dark mode
-* Add option to delete mailbox data when mailbox is delete
-* Allow days/hours of backups and updates to be configurable
-* backup cleaner: fix issue where referenced backups where not counted against time periods
-* route53: fix issue where verification failed if user had more than 100 zones
-* rework task workers to run them in a separate cgroup
-* backups: now much faster thanks to reworking of task worker
-* When custom fallback cert is set, make sure it's used over LE certs
-* mongodb: update to MongoDB 4.0.19
-* List groups ordered by name
-* Invite links are now valid for a week
-* Update release GPG key
-* Add pre-defined variables ($CLOUDRON_APPID) for better post install messages
-* filemanager: show folder first
-

README.md

@@ -48,8 +48,18 @@ the dashboard, database addons, graph container, base image etc. Cloudron also r
 on external services such as the App Store for apps to be installed. As such, don't
 clone this repo and npm install and expect something to work.
 
-## Support
+## Documentation
 
 * [Documentation](https://cloudron.io/documentation/)
-* [Forum](https://forum.cloudron.io/)
+
+## Related repos
+
+The [base image repo](https://git.cloudron.io/cloudron/docker-base-image) is the parent image of all
+the containers in the Cloudron.
+
+## Community
+
+* [Chat](https://chat.cloudron.io)
+* [Forum](https://forum.cloudron.io/)
+* [Support](mailto:support@cloudron.io)
 

baseimage/initializeBaseUbuntuImage.sh

@@ -4,7 +4,8 @@ set -euv -o pipefail
 
 readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-readonly arg_infraversionpath="${SOURCE_DIR}/../src"
+readonly arg_provider="${1:-generic}"
+readonly arg_infraversionpath="${SOURCE_DIR}/${2:-}"
 
 function die {
     echo $1
@@ -13,9 +14,6 @@ function die {
 
 export DEBIAN_FRONTEND=noninteractive
 
-readonly ubuntu_codename=$(lsb_release -cs)
-readonly ubuntu_version=$(lsb_release -rs)
-
 # hold grub since updating it breaks on some VPS providers. also, dist-upgrade will trigger it
 apt-mark hold grub* >/dev/null
 apt-get -o Dpkg::Options::="--force-confdef" update -y
@@ -29,6 +27,8 @@ debconf-set-selections <<< 'mysql-server mysql-server/root_password_again passwo
 
 # this enables automatic security upgrades (https://help.ubuntu.com/community/AutomaticSecurityUpdates)
 # resolvconf is needed for unbound to work property after disabling systemd-resolved in 18.04
+ubuntu_version=$(lsb_release -rs)
+ubuntu_codename=$(lsb_release -cs)
 gpg_package=$([[ "${ubuntu_version}" == "16.04" ]] && echo "gnupg" || echo "gpg")
 apt-get -y install \
     acl \
@@ -44,6 +44,7 @@ apt-get -y install \
     linux-generic \
     logrotate \
     mysql-server-5.7 \
+    nginx-full \
     openssh-server \
     pwgen \
     resolvconf \
@@ -53,12 +54,6 @@ apt-get -y install \
     unbound \
     xfsprogs
 
-echo "==> installing nginx for xenial for TLSv3 support"
-curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-1~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-# apt install with install deps (as opposed to dpkg -i)
-apt install -y /tmp/nginx.deb
-rm /tmp/nginx.deb
-
 # on some providers like scaleway the sudo file is changed and we want to keep the old one
 apt-get -o Dpkg::Options::="--force-confold" install -y sudo
 
@@ -68,7 +63,7 @@ cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upg
 
 echo "==> Installing node.js"
 mkdir -p /usr/local/node-10.18.1
-curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxf - --strip-components=1 -C /usr/local/node-10.18.1
+curl -sL https://nodejs.org/dist/v10.18.1/node-v10.18.1-linux-x64.tar.gz | tar zxvf - --strip-components=1 -C /usr/local/node-10.18.1
 ln -sf /usr/local/node-10.18.1/bin/node /usr/bin/node
 ln -sf /usr/local/node-10.18.1/bin/npm /usr/bin/npm
 apt-get install -y python   # Install python which is required for npm rebuild
@@ -116,7 +111,7 @@ for image in ${images}; do
 done
 
 echo "==> Install collectd"
-if ! apt-get install -y libcurl3-gnutls collectd collectd-utils; then
+if ! apt-get install -y collectd collectd-utils; then
     # FQDNLookup is true in default debian config. The box code has a custom collectd.conf that fixes this
     echo "Failed to install collectd. Presumably because of http://mailman.verplant.org/pipermail/collectd/2015-March/006491.html"
     sed -e 's/^FQDNLookup true/FQDNLookup false/' -i /etc/collectd/collectd.conf

box.js

@@ -2,60 +2,57 @@
 
 'use strict';
 
+// prefix all output with a timestamp
+// debug() already prefixes and uses process.stderr NOT console.*
+['log', 'info', 'warn', 'debug', 'error'].forEach(function (log) {
+    var orig = console[log];
+    console[log] = function () {
+        orig.apply(console, [new Date().toISOString()].concat(Array.prototype.slice.call(arguments)));
+    };
+});
+
+require('supererror')({ splatchError: true });
+
 let async = require('async'),
+    constants = require('./src/constants.js'),
     dockerProxy = require('./src/dockerproxy.js'),
-    fs = require('fs'),
     ldap = require('./src/ldap.js'),
-    paths = require('./src/paths.js'),
-    server = require('./src/server.js'),
-    util = require('util');
+    server = require('./src/server.js');
 
-const NOOP_CALLBACK = function () { };
-
-function setupLogging(callback) {
-    if (process.env.BOX_ENV === 'test') return callback();
-
-    fs.open(paths.BOX_LOG_FILE, 'a', function (error, fd) {
-        if (error) return callback(error);
-
-        require('debug').log = function (...args) {
-            fs.appendFileSync(fd, util.format(...args) + '\n');
-        };
-
-        callback();
-    });
-}
+console.log();
+console.log('==========================================');
+console.log(`           Cloudron ${constants.VERSION}  `);
+console.log('==========================================');
+console.log();
 
 async.series([
-    setupLogging,
     server.start,
     ldap.start,
     dockerProxy.start
 ], function (error) {
     if (error) {
-        console.log('Error starting server', error);
+        console.error('Error starting server', error);
         process.exit(1);
     }
-
-    const debug = require('debug')('box:box'); // require this here so that logging handler is already setup
-
-    process.on('SIGINT', function () {
-        debug('Received SIGINT. Shutting down.');
-
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
-        setTimeout(process.exit.bind(process), 3000);
-    });
-
-    process.on('SIGTERM', function () {
-        debug('Received SIGTERM. Shutting down.');
-
-        server.stop(NOOP_CALLBACK);
-        ldap.stop(NOOP_CALLBACK);
-        dockerProxy.stop(NOOP_CALLBACK);
-        setTimeout(process.exit.bind(process), 3000);
-    });
-
-    console.log(`Cloudron is up and running. Logs are at ${paths.BOX_LOG_FILE}`); // this goes to journalctl
+    console.log('Cloudron is up and running');
+});
+
+var NOOP_CALLBACK = function () { };
+
+process.on('SIGINT', function () {
+    console.log('Received SIGINT. Shutting down.');
+
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    dockerProxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
+});
+
+process.on('SIGTERM', function () {
+    console.log('Received SIGTERM. Shutting down.');
+
+    server.stop(NOOP_CALLBACK);
+    ldap.stop(NOOP_CALLBACK);
+    dockerProxy.stop(NOOP_CALLBACK);
+    setTimeout(process.exit.bind(process), 3000);
 });

migrations/20160208164735-groups-add-admin.js

@@ -12,6 +12,8 @@ exports.up = function(db, callback) {
 			db.all('SELECT * FROM users WHERE admin=1', function (error, results) {
 				if (error) return done(error);
 
+				console.dir(results);
+
 				async.eachSeries(results, function (r, next) {
 					db.runSql('INSERT INTO groupMembers (groupId, userId) VALUES (?, ?)', [ ADMIN_GROUP_ID, r.id ], next);
 				}, done);

migrations/20200330233442-users-add-resetTokenCreationTime.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN resetTokenCreationTime', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200331164304-apps-alter-mailboxDomain.js

@@ -1,28 +0,0 @@
-'use strict';
-
-let async = require('async');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY mailboxDomain VARCHAR(128)', [], function (error) { // make it nullable
-        if (error) console.error(error);
-
-        // clear mailboxName/Domain for apps that do not use mail addons
-        db.all('SELECT * FROM apps', function (error, apps) {
-            if (error) return callback(error);
-
-            async.eachSeries(apps, function (app, iteratorDone) {
-                var manifest = JSON.parse(app.manifestJson);
-                if (manifest.addons['sendmail'] || manifest.addons['recvmail']) return iteratorDone();
-
-                db.runSql('UPDATE apps SET mailboxName=?, mailboxDomain=? WHERE id=?', [ null, null, app.id ], iteratorDone);
-            }, callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps MODIFY manifestJson VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200417235435-mailboxes-add-membersOnly.js

@@ -1,17 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE mailboxes ADD COLUMN membersOnly BOOLEAN DEFAULT 0', function (error) {
-        if (error) return callback(error);
-
-        callback();
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN membersOnly', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20200420013715-mailboxes-add-aliasDomain.js

@@ -1,28 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD COLUMN aliasDomain VARCHAR(128)'),
-        function setAliasDomain(done) {
-            db.all('SELECT * FROM mailboxes', function (error, mailboxes) {
-                async.eachSeries(mailboxes, function (mailbox, iteratorDone) {
-                    if (!mailbox.aliasTarget) return iteratorDone();
-
-                    db.runSql('UPDATE mailboxes SET aliasDomain=? WHERE name=? AND domain=?', [ mailbox.domain, mailbox.name, mailbox.domain ], iteratorDone);
-                }, done);
-            });
-        },
-        db.runSql.bind(db, 'ALTER TABLE mailboxes ADD CONSTRAINT mailboxes_aliasDomain_constraint FOREIGN KEY(aliasDomain) REFERENCES mail(domain)'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes CHANGE aliasTarget aliasName VARCHAR(128)')
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP FOREIGN KEY mailboxes_aliasDomain_constraint'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes DROP COLUMN aliasDomain'),
-        db.runSql.bind(db, 'ALTER TABLE mailboxes CHANGE aliasName aliasTarget VARCHAR(128)')
-    ], callback);
-};

migrations/20200427044800-apps-add-servicesConfigJson.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN servicesConfigJson TEXT', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN servicesConfigJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200430035310-apps-add-bindsJson.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE apps ADD COLUMN bindsJson TEXT', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE apps DROP COLUMN bindsJson', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200512172301-settings-backup-encryption.js

@@ -1,35 +0,0 @@
-'use strict';
-
-const backups = require('../src/backups.js'),
-    fs = require('fs');
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.key) {
-            backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.key);
-            backups.cleanupCacheFilesSync();
-
-            fs.writeFileSync('/home/yellowtent/platformdata/BACKUP_PASSWORD',
-                'This file contains your Cloudron backup password.\nBefore Cloudron v5.2, this was saved in the database.' +
-                'From Cloudron 5.2, this password is not required anymore. We generate strong keys based off this password and use those keys to encrypt the backups.\n' +
-                'This means that the password is only required at decryption/restore time.\n\n' +
-                'This file can be safely removed and only exists for the off-chance that you do not remember your backup password.\n\n' +
-                `Password: ${backupConfig.key}\n`,
-                'utf8');
-
-        } else {
-            backupConfig.encryption = null;
-        }
-
-        delete backupConfig.key;
-
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20200514043113-backups-rename-version-to-packageVersion.js

@@ -1,15 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups CHANGE version packageVersion VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups CHANGE packageVersion version VARCHAR(128) NOT NULL', [], function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200514045746-backups-add-encryptionVersion.js

@@ -1,24 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups ADD COLUMN encryptionVersion INTEGER', function (error) {
-        if (error) return callback(error);
-
-        db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-            if (error || results.length === 0) return callback(error);
-
-            var backupConfig = JSON.parse(results[0].value);
-            if (!backupConfig.encryption) return callback(null);
-
-            // mark old encrypted backups as v1
-            db.runSql('UPDATE backups SET encryptionVersion=1', callback);
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups DROP COLUMN encryptionVersion', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200514231149-backups-retentionSecs-to-retentionPolicy.js

@@ -1,18 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        backupConfig.retentionPolicy = { keepWithinSecs: backupConfig.retentionSecs };
-        delete backupConfig.retentionSecs;
-
-        // mark old encrypted backups as v1
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20200528003052-settings-backup-s3-path-style.js

@@ -1,18 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.provider !== 'minio' && backupConfig.provider !== 's3-v4-compat') return callback();
-
-        backupConfig.s3ForcePathStyle = true; // usually minio is self-hosted. s3 v4 compat, we don't know
-
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20200530194049-appPasswords-change-unique-constraint.js

@@ -1,17 +0,0 @@
-'use strict';
-
-var async = require('async');
-
-exports.up = function(db, callback) {
-
-    // http://stackoverflow.com/questions/386294/what-is-the-maximum-length-of-a-valid-email-address
-
-    async.series([
-        db.runSql.bind(db, 'ALTER TABLE appPasswords DROP INDEX name'),
-        db.runSql.bind(db, 'ALTER TABLE appPasswords ADD CONSTRAINT appPasswords_name_userId_identifier UNIQUE (name, userId, identifier)'),
-    ], callback);
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/20200604111448-userGroups-add-source.js

@@ -1,17 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE userGroups ADD COLUMN source VARCHAR(128) DEFAULT ""', function (error) {
-        if (error) return callback(error);
-
-        callback();
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE userGroups DROP COLUMN source', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};
-

migrations/20200614181936-backups-add-identifier.js

@@ -1,38 +0,0 @@
-'use strict';
-
-const async = require('async');
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE backups ADD COLUMN identifier VARCHAR(128)', function (error) {
-        if (error) return callback(error);
-
-
-        db.all('SELECT * FROM backups', function (error, backups) {
-            if (error) return callback(error);
-
-            async.eachSeries(backups, function (backup, next) {
-                let identifier = 'unknown';
-
-                if (backup.type === 'box') {
-                    identifier = 'box';
-                } else {
-                    const match = backup.id.match(/app_(.+?)_.+/);
-                    if (match) identifier = match[1];
-                }
-
-                db.runSql('UPDATE backups SET identifier=? WHERE id=?', [ identifier, backup.id ], next);
-            }, function (error) {
-                if (error) return callback(error);
-
-                db.runSql('ALTER TABLE backups MODIFY COLUMN identifier VARCHAR(128) NOT NULL', callback);
-            });
-        });
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE backups DROP COLUMN identifier', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200709224148-users-rename-modifiedAt-to-ts.js

@@ -1,16 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.runSql('ALTER TABLE users ADD COLUMN ts TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP', function (error) {
-        if (error) console.error(error);
-
-        db.runSql('ALTER TABLE users DROP COLUMN modifiedAt', callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    db.runSql('ALTER TABLE users DROP COLUMN ts', function (error) {
-        if (error) console.error(error);
-        callback(error);
-    });
-};

migrations/20200728232132-settings-set-backups-schedulePattern.js

@@ -1,29 +0,0 @@
-'use strict';
-
-exports.up = function(db, callback) {
-    db.all('SELECT value FROM settings WHERE name="backup_config"', function (error, results) {
-        if (error || results.length === 0) return callback(error);
-
-        var backupConfig = JSON.parse(results[0].value);
-        if (backupConfig.intervalSecs === 6 * 60 * 60) { // every 6 hours
-            backupConfig.schedulePattern = '00 00 5,11,17,23 * * *';
-        } else if (backupConfig.intervalSecs === 12 * 60 * 60) { // every 12 hours
-            backupConfig.schedulePattern = '00 00 5,17 * * *';
-        } else if (backupConfig.intervalSecs === 24 * 60 * 60) { // every day
-            backupConfig.schedulePattern = '00 00 23 * * *';
-        } else if (backupConfig.intervalSecs === 3 * 24 * 60 * 60) { // every 3 days (based on day)
-            backupConfig.schedulePattern = '00 00 23 * * 1,3,5';
-        } else if (backupConfig.intervalSecs === 7 * 24 * 60 * 60) { // every week (saturday)
-            backupConfig.schedulePattern = '00 00 23 * * 6';
-        } else { // default to everyday
-            backupConfig.schedulePattern = '00 00 23 * * *';
-        }
-
-        delete backupConfig.intervalSecs;
-        db.runSql('UPDATE settings SET value=? WHERE name="backup_config"', [ JSON.stringify(backupConfig) ], callback);
-    });
-};
-
-exports.down = function(db, callback) {
-    callback();
-};

migrations/schema.sql

@@ -21,23 +21,19 @@ CREATE TABLE IF NOT EXISTS users(
     password VARCHAR(1024) NOT NULL,
     salt VARCHAR(512) NOT NULL,
     createdAt VARCHAR(512) NOT NULL,
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    modifiedAt VARCHAR(512) NOT NULL,
     displayName VARCHAR(512) DEFAULT "",
     fallbackEmail VARCHAR(512) DEFAULT "",
     twoFactorAuthenticationSecret VARCHAR(128) DEFAULT "",
     twoFactorAuthenticationEnabled BOOLEAN DEFAULT false,
     source VARCHAR(128) DEFAULT "",
     role VARCHAR(32),
-    resetToken VARCHAR(128) DEFAULT "",
-    resetTokenCreationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    active BOOLEAN DEFAULT 1,
 
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS userGroups(
     id VARCHAR(128) NOT NULL UNIQUE,
     name VARCHAR(254) NOT NULL UNIQUE,
-    source VARCHAR(128) DEFAULT "",
     PRIMARY KEY(id));
 
 CREATE TABLE IF NOT EXISTS groupMembers(
@@ -80,15 +76,13 @@ CREATE TABLE IF NOT EXISTS apps(
     reverseProxyConfigJson TEXT, // { robotsTxt, csp }
     enableBackup BOOLEAN DEFAULT 1, // misnomer: controls automatic daily backups
     enableAutomaticUpdate BOOLEAN DEFAULT 1,
-    mailboxName VARCHAR(128), // mailbox of this app
-    mailboxDomain VARCHAR(128), // mailbox domain of this apps
+    mailboxName VARCHAR(128), // mailbox of this app. default allocated as '.app'
+    mailboxDomain VARCHAR(128) NOT NULL, // mailbox domain of this apps
     label VARCHAR(128),     // display name
     tagsJson VARCHAR(2048), // array of tags
     dataDir VARCHAR(256) UNIQUE,
     taskId INTEGER, // current task
     errorJson TEXT,
-    bindsJson TEXT, // bind mounts
-    servicesConfigJson TEXT, // app services configuration
 
     FOREIGN KEY(mailboxDomain) REFERENCES domains(domain),
     FOREIGN KEY(taskId) REFERENCES tasks(id),
@@ -123,10 +117,8 @@ CREATE TABLE IF NOT EXISTS appEnvVars(
 CREATE TABLE IF NOT EXISTS backups(
     id VARCHAR(128) NOT NULL,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    packageVersion VARCHAR(128) NOT NULL, /* app version or box version */
-    encryptionVersion INTEGER, /* when null, unencrypted backup */
+    version VARCHAR(128) NOT NULL, /* app version or box version */
     type VARCHAR(16) NOT NULL, /* 'box' or 'app' */
-    identifier VARCHAR(128) NOT NULL, /* 'box' or the app id */
     dependsOn TEXT, /* comma separate list of objects this backup depends on */
     state VARCHAR(16) NOT NULL,
     manifestJson TEXT, /* to validate if the app can be installed in this version of box */
@@ -182,15 +174,12 @@ CREATE TABLE IF NOT EXISTS mailboxes(
     name VARCHAR(128) NOT NULL,
     type VARCHAR(16) NOT NULL, /* 'mailbox', 'alias', 'list' */
     ownerId VARCHAR(128) NOT NULL, /* user id */
-    aliasName VARCHAR(128), /* the target name type is an alias */
-    aliasDomain VARCHAR(128), /* the target domain */
+    aliasTarget VARCHAR(128), /* the target name type is an alias */
     membersJson TEXT, /* members of a group. fully qualified */
-    membersOnly BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     domain VARCHAR(128),
 
     FOREIGN KEY(domain) REFERENCES mail(domain),
-    FOREIGN KEY(aliasDomain) REFERENCES mail(domain),
     UNIQUE (name, domain));
 
 CREATE TABLE IF NOT EXISTS subdomains(
@@ -222,7 +211,7 @@ CREATE TABLE IF NOT EXISTS notifications(
     message TEXT,
     acknowledged BOOLEAN DEFAULT false,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    UNIQUE KEY appPasswords_name_appId_identifier (name, userId, identifier),
+
     PRIMARY KEY (id)
 );
 
@@ -234,7 +223,7 @@ CREATE TABLE IF NOT EXISTS appPasswords(
     hashedPassword VARCHAR(1024) NOT NULL,
     creationTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     FOREIGN KEY(userId) REFERENCES users(id),
-
+    UNIQUE (name, userId),
     PRIMARY KEY (id)
 );
 

package.json

@@ -18,33 +18,32 @@
     "@google-cloud/storage": "^2.5.0",
     "@sindresorhus/df": "git+https://github.com/cloudron-io/df.git#type",
     "async": "^2.6.3",
-    "aws-sdk": "^2.685.0",
+    "aws-sdk": "^2.610.0",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.5.0",
+    "cloudron-manifestformat": "^4.0.0",
     "connect": "^3.7.0",
-    "connect-lastmile": "^2.0.0",
+    "connect-lastmile": "^1.2.2",
     "connect-timeout": "^1.9.0",
     "cookie-session": "^1.4.0",
     "cron": "^1.8.2",
-    "db-migrate": "^0.11.11",
+    "db-migrate": "^0.11.6",
     "db-migrate-mysql": "^1.1.10",
     "debug": "^4.1.1",
     "dockerode": "^2.5.8",
     "ejs": "^2.6.1",
-    "ejs-cli": "^2.2.0",
+    "ejs-cli": "^2.1.1",
     "express": "^4.17.1",
-    "js-yaml": "^3.14.0",
+    "js-yaml": "^3.13.1",
     "json": "^9.0.6",
     "ldapjs": "^1.0.2",
     "lodash": "^4.17.15",
     "lodash.chunk": "^4.2.0",
-    "mime": "^2.4.6",
-    "moment": "^2.26.0",
-    "moment-timezone": "^0.5.31",
-    "morgan": "^1.10.0",
+    "mime": "^2.4.4",
+    "moment-timezone": "^0.5.27",
+    "morgan": "^1.9.1",
     "multiparty": "^4.2.1",
     "mysql": "^2.18.1",
-    "nodemailer": "^6.4.6",
+    "nodemailer": "^6.4.2",
     "nodemailer-smtp-transport": "^2.7.4",
     "once": "^1.4.0",
     "parse-links": "^0.1.0",
@@ -52,33 +51,34 @@
     "progress-stream": "^2.0.0",
     "proxy-middleware": "^0.15.0",
     "qrcode": "^1.4.4",
-    "readdirp": "^3.4.0",
-    "request": "^2.88.2",
+    "readdirp": "^3.3.0",
+    "request": "^2.88.0",
     "rimraf": "^2.6.3",
     "s3-block-read-stream": "^0.5.0",
-    "safetydance": "^1.1.1",
+    "safetydance": "^1.0.0",
     "semver": "^6.1.1",
     "showdown": "^1.9.1",
     "speakeasy": "^2.0.0",
     "split": "^1.0.1",
-    "superagent": "^5.2.2",
+    "superagent": "^5.2.1",
+    "supererror": "^0.7.2",
     "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error",
-    "tar-stream": "^2.1.2",
+    "tar-stream": "^2.1.0",
     "tldjs": "^2.3.1",
-    "underscore": "^1.10.2",
+    "underscore": "^1.9.2",
     "uuid": "^3.4.0",
     "validator": "^11.0.0",
-    "ws": "^7.3.0",
+    "ws": "^7.2.1",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
     "expect.js": "*",
-    "hock": "^1.4.1",
-    "js2xmlparser": "^4.0.1",
+    "hock": "^1.3.3",
+    "js2xmlparser": "^4.0.0",
     "mocha": "^6.1.4",
     "mock-aws-s3": "git+https://github.com/cloudron-io/mock-aws-s3.git",
     "nock": "^10.0.6",
-    "node-sass": "^4.14.1",
+    "node-sass": "^4.12.0",
     "recursive-readdir": "^2.2.2"
   },
   "scripts": {

scripts/cloudron-setup

@@ -41,14 +41,16 @@ if systemctl -q is-active box; then
 fi
 
 initBaseImage="true"
-provider="generic"
+# provisioning data
+provider=""
 requestedVersion=""
 apiServerOrigin="https://api.cloudron.io"
 webServerOrigin="https://cloudron.io"
 sourceTarballUrl=""
 rebootServer="true"
+license=""
 
-args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,skip-baseimage-init,provider:,version:,env:,skip-reboot,license:" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -65,6 +67,7 @@ while true; do
             webServerOrigin="https://staging.cloudron.io"
         fi
         shift 2;;
+    --license) license="$2"; shift 2;;
     --skip-baseimage-init) initBaseImage="false"; shift;;
     --skip-reboot) rebootServer="false"; shift;;
     --) break;;
@@ -88,6 +91,48 @@ fi
 # Can only write after we have confirmed script has root access
 echo "Running cloudron-setup with args : $@" > "${LOG_FILE}"
 
+# validate arguments in the absence of data
+readonly AVAILABLE_PROVIDERS="azure, caas, cloudscale, contabo, digitalocean, ec2, exoscale, gce, hetzner, interox, lightsail, linode, netcup, ovh, rosehosting, scaleway, skysilk, time4vps, upcloud, vultr or generic"
+if [[ -z "${provider}" ]]; then
+    echo "--provider is required ($AVAILABLE_PROVIDERS)"
+    exit 1
+elif [[  \
+            "${provider}" != "ami" && \
+            "${provider}" != "azure" && \
+            "${provider}" != "azure-image" && \
+            "${provider}" != "caas" && \
+            "${provider}" != "cloudscale" && \
+            "${provider}" != "contabo" && \
+            "${provider}" != "digitalocean" && \
+            "${provider}" != "digitalocean-mp" && \
+            "${provider}" != "ec2" && \
+            "${provider}" != "exoscale" && \
+            "${provider}" != "gce" && \
+            "${provider}" != "hetzner" && \
+            "${provider}" != "interox" && \
+            "${provider}" != "interox-image" && \
+            "${provider}" != "lightsail" && \
+            "${provider}" != "linode" && \
+            "${provider}" != "linode-oneclick" && \
+            "${provider}" != "linode-stackscript" && \
+            "${provider}" != "netcup" && \
+            "${provider}" != "netcup-image" && \
+            "${provider}" != "ovh" && \
+            "${provider}" != "rosehosting" && \
+            "${provider}" != "scaleway" && \
+            "${provider}" != "skysilk" && \
+            "${provider}" != "skysilk-image" && \
+            "${provider}" != "time4vps" && \
+            "${provider}" != "time4vps-image" && \
+            "${provider}" != "upcloud" && \
+            "${provider}" != "upcloud-image" && \
+            "${provider}" != "vultr" && \
+            "${provider}" != "generic" \
+        ]]; then
+    echo "--provider must be one of: $AVAILABLE_PROVIDERS"
+    exit 1
+fi
+
 echo ""
 echo "##############################################"
 echo "         Cloudron Setup (${requestedVersion:-latest})"
@@ -106,6 +151,12 @@ if [[ "${initBaseImage}" == "true" ]]; then
         exit 1
     fi
 
+    echo "=> Ensure required apt sources"
+    if ! add-apt-repository universe &>> "${LOG_FILE}"; then
+        echo "Could not add required apt sources (for nginx-full). See ${LOG_FILE}"
+        exit 1
+    fi
+
     echo "=> Updating apt and installing script dependencies"
     if ! apt-get update &>> "${LOG_FILE}"; then
         echo "Could not update package repositories. See ${LOG_FILE}"
@@ -145,19 +196,20 @@ fi
 
 if [[ "${initBaseImage}" == "true" ]]; then
     echo -n "=> Installing base dependencies and downloading docker images (this takes some time) ..."
-    # initializeBaseUbuntuImage.sh args (provider, infraversion path) are only to support installation of pre 5.3 Cloudrons
-    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "generic" "../src" &>> "${LOG_FILE}"; then
+    if ! /bin/bash "${box_src_tmp_dir}/baseimage/initializeBaseUbuntuImage.sh" "${provider}" "../src" &>> "${LOG_FILE}"; then
         echo "Init script failed. See ${LOG_FILE} for details"
         exit 1
     fi
     echo ""
 fi
 
-# The provider flag is still used for marketplace images
+# NOTE: this install script only supports 4.2 and above
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER
 
+[[ -n "${license}" ]] && echo -n "$license" > /etc/cloudron/LICENSE
+
 if ! /bin/bash "${box_src_tmp_dir}/scripts/installer.sh" &>> "${LOG_FILE}"; then
     echo "Failed to install cloudron. See ${LOG_FILE} for details"
     exit 1
@@ -169,13 +221,13 @@ mysql -uroot -ppassword -e "REPLACE INTO box.settings (name, value) VALUES ('web
 echo -n "=> Waiting for cloudron to be ready (this takes some time) ..."
 while true; do
     echo -n "."
-    if status=$($curl -s -f "http://localhost:3000/api/v1/cloudron/status" 2>/dev/null); then
+    if status=$($curl -q -f "http://localhost:3000/api/v1/cloudron/status" 2>/dev/null); then
         break # we are up and running
     fi
     sleep 10
 done
 
-if ! ip=$(curl -s --fail --connect-timeout 2 --max-time 2 https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
+if ! ip=$(curl --fail --connect-timeout 2 --max-time 2 -q https://api.cloudron.io/api/v1/helper/public_ip | sed -n -e 's/.*"ip": "\(.*\)"/\1/p'); then
     ip='<IP>'
 fi
 echo -e "\n\n${GREEN}Visit https://${ip} and accept the self-signed certificate to finish setup.${DONE}\n"

scripts/cloudron-support

@@ -13,7 +13,7 @@ HELP_MESSAGE="
 This script collects diagnostic information to help debug server related issues
 
  Options:
-   --owner-login   Login as owner
+   --admin-login   Login as administrator
    --enable-ssh    Enable SSH access for the Cloudron support team
    --help          Show this message
 "
@@ -26,7 +26,7 @@ fi
 
 enableSSH="false"
 
-args=$(getopt -o "" -l "help,enable-ssh,admin-login,owner-login" -n "$0" -- "$@")
+args=$(getopt -o "" -l "help,enable-ssh,admin-login" -n "$0" -- "$@")
 eval set -- "${args}"
 
 while true; do
@@ -34,9 +34,6 @@ while true; do
     --help) echo -e "${HELP_MESSAGE}"; exit 0;;
     --enable-ssh) enableSSH="true"; shift;;
     --admin-login)
-        # fall through
-        ;&
-    --owner-login)
         admin_username=$(mysql -NB -uroot -ppassword -e "SELECT username FROM box.users WHERE role='owner' LIMIT 1" 2>/dev/null)
         admin_password=$(pwgen -1s 12)
         ghost_file=/home/yellowtent/platformdata/cloudron_ghost.json
@@ -94,7 +91,7 @@ echo -e $LINE"Backup stats (possibly misleading)"$LINE >> $OUT
 du -hcsL /var/backups/* &>> $OUT || true
 
 echo -e $LINE"System daemon status"$LINE >> $OUT
-systemctl status --lines=100 box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT
+systemctl status --lines=100 cloudron.target box mysql unbound cloudron-syslog nginx collectd docker &>> $OUT
 
 echo -e $LINE"Box logs"$LINE >> $OUT
 tail -n 100 /home/yellowtent/platformdata/logs/box.log &>> $OUT
@@ -112,7 +109,7 @@ if [[ "${enableSSH}" == "true" ]]; then
     permit_root_login=$(grep -q ^PermitRootLogin.*yes /etc/ssh/sshd_config && echo "yes" || echo "no")
 
     # support.js uses similar logic
-    if [[ -d /home/ubuntu ]]; then
+    if $(grep -q "ec2\|lightsail\|ami" /etc/cloudron/PROVIDER); then
         ssh_user="ubuntu"
         keys_file="/home/ubuntu/.ssh/authorized_keys"
     else

scripts/installer.sh

@@ -11,8 +11,9 @@ if [[ ${EUID} -ne 0 ]]; then
     exit 1
 fi
 
-readonly user=yellowtent
-readonly box_src_dir=/home/${user}/box
+readonly USER=yellowtent
+readonly BOX_SRC_DIR=/home/${USER}/box
+readonly BASE_DATA_DIR=/home/${USER}
 
 readonly curl="curl --fail --connect-timeout 20 --retry 10 --retry-delay 2 --max-time 2400"
 readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -23,8 +24,6 @@ readonly ubuntu_codename=$(lsb_release -cs)
 
 readonly is_update=$(systemctl is-active box && echo "yes" || echo "no")
 
-echo "==> installer: Updating from $(cat $box_src_dir/VERSION) to $(cat $box_src_tmp_dir/VERSION) <=="
-
 echo "==> installer: updating docker"
 
 if [[ $(docker version --format {{.Client.Version}}) != "18.09.2" ]]; then
@@ -57,15 +56,6 @@ if [[ $(docker version --format {{.Client.Version}}) != "18.09.2" ]]; then
     rm /tmp/containerd.deb /tmp/docker-ce-cli.deb /tmp/docker.deb
 fi
 
-readonly nginx_version=$(nginx -v 2>&1)
-if [[ "${nginx_version}" != *"1.18."* ]]; then
-    echo "==> installer: installing nginx 1.18"
-    curl -sL http://nginx.org/packages/ubuntu/pool/nginx/n/nginx/nginx_1.18.0-1~${ubuntu_codename}_amd64.deb -o /tmp/nginx.deb
-    # apt install with install deps (as opposed to dpkg -i)
-    apt install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes /tmp/nginx.deb
-    rm /tmp/nginx.deb
-fi
-
 echo "==> installer: updating node"
 if [[ "$(node --version)" != "v10.18.1" ]]; then
     mkdir -p /usr/local/node-10.18.1
@@ -119,22 +109,22 @@ while [[ ! -f "${CLOUDRON_SYSLOG}" || "$(${CLOUDRON_SYSLOG} --version)" != ${CLO
     sleep 5
 done
 
-if ! id "${user}" 2>/dev/null; then
-    useradd "${user}" -m
+if ! id "${USER}" 2>/dev/null; then
+    useradd "${USER}" -m
 fi
 
 if [[ "${is_update}" == "yes" ]]; then
-    echo "==> installer: stop box service for update"
-    ${box_src_dir}/setup/stop.sh
+    echo "==> installer: stop cloudron.target service for update"
+    ${BOX_SRC_DIR}/setup/stop.sh
 fi
 
 # ensure we are not inside the source directory, which we will remove now
 cd /root
 
 echo "==> installer: switching the box code"
-rm -rf "${box_src_dir}"
-mv "${box_src_tmp_dir}" "${box_src_dir}"
-chown -R "${user}:${user}" "${box_src_dir}"
+rm -rf "${BOX_SRC_DIR}"
+mv "${box_src_tmp_dir}" "${BOX_SRC_DIR}"
+chown -R "${USER}:${USER}" "${BOX_SRC_DIR}"
 
 echo "==> installer: calling box setup script"
-"${box_src_dir}/setup/start.sh"
+"${BOX_SRC_DIR}/setup/start.sh"

setup/start.sh

@@ -20,11 +20,6 @@ readonly ubuntu_version=$(lsb_release -rs)
 
 cp -f "${script_dir}/../scripts/cloudron-support" /usr/bin/cloudron-support
 
-# this needs to match the cloudron/base:2.0.0 gid
-if ! getent group media; then
-    addgroup --gid 500 --system media
-fi
-
 echo "==> Configuring docker"
 cp "${script_dir}/start/docker-cloudron-app.apparmor" /etc/apparmor.d/docker-cloudron-app
 systemctl enable apparmor
@@ -61,7 +56,6 @@ mkdir -p "${BOX_DATA_DIR}/profileicons"
 mkdir -p "${BOX_DATA_DIR}/certs"
 mkdir -p "${BOX_DATA_DIR}/acme" # acme keys
 mkdir -p "${BOX_DATA_DIR}/mail/dkim"
-mkdir -p "${BOX_DATA_DIR}/well-known" # .well-known documents
 
 # ensure backups folder exists and is writeable
 mkdir -p /var/backups
@@ -85,9 +79,6 @@ systemctl daemon-reload
 systemctl restart systemd-journald
 setfacl -n -m u:${USER}:r /var/log/journal/*/system.journal
 
-# Give user access to nginx logs (uses adm group)
-usermod -a -G adm ${USER}
-
 echo "==> Setting up unbound"
 # DO uses Google nameservers by default. This causes RBL queries to fail (host 2.0.0.127.zen.spamhaus.org)
 # We do not use dnsmasq because it is not a recursive resolver and defaults to the value in the interfaces file (which is Google DNS!)
@@ -100,13 +91,11 @@ unbound-anchor -a /var/lib/unbound/root.key
 
 echo "==> Adding systemd services"
 cp -r "${script_dir}/start/systemd/." /etc/systemd/system/
-systemctl disable cloudron.target || true
-rm -f /etc/systemd/system/cloudron.target
 [[ "${ubuntu_version}" == "16.04" ]] && sed -e 's/MemoryMax/MemoryLimit/g' -i /etc/systemd/system/box.service
 systemctl daemon-reload
 systemctl enable unbound
 systemctl enable cloudron-syslog
-systemctl enable box
+systemctl enable cloudron.target
 systemctl enable cloudron-firewall
 
 # update firewall rules
@@ -155,15 +144,8 @@ cp "${script_dir}/start/nginx/mime.types" "${PLATFORM_DATA_DIR}/nginx/mime.types
 if ! grep -q "^Restart=" /etc/systemd/system/multi-user.target.wants/nginx.service; then
     # default nginx service file does not restart on crash
     echo -e "\n[Service]\nRestart=always\n" >> /etc/systemd/system/multi-user.target.wants/nginx.service
+    systemctl daemon-reload
 fi
-
-# worker_rlimit_nofile in nginx config can be max this number
-mkdir -p /etc/systemd/system/nginx.service.d
-if ! grep -q "^LimitNOFILE=" /etc/systemd/system/nginx.service.d/cloudron.conf; then
-    echo -e "[Service]\nLimitNOFILE=16384\n" > /etc/systemd/system/nginx.service.d/cloudron.conf
-fi
-
-systemctl daemon-reload
 systemctl start nginx
 
 # restart mysql to make sure it has latest config
@@ -188,11 +170,9 @@ readonly mysql_root_password="password"
 mysqladmin -u root -ppassword password password # reset default root password
 mysql -u root -p${mysql_root_password} -e 'CREATE DATABASE IF NOT EXISTS box'
 
-# set HOME explicity, because it's not set when the installer calls it. this is done because
-# paths.js uses this env var and some of the migrate code requires box code
 echo "==> Migrating data"
 cd "${BOX_SRC_DIR}"
-if ! HOME=${HOME_DIR} BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then
+if ! BOX_ENV=cloudron DATABASE_URL=mysql://root:${mysql_root_password}@127.0.0.1/box "${BOX_SRC_DIR}/node_modules/.bin/db-migrate" up; then
     echo "DB migration failed"
     exit 1
 fi
@@ -211,9 +191,6 @@ fi
 echo "==> Cleaning up stale redis directories"
 find "${APPS_DATA_DIR}" -maxdepth 2 -type d -name redis -exec rm -rf {} +
 
-echo "==> Cleaning up old logs"
-rm -f /home/yellowtent/platformdata/logs/*/*.log.* || true
-
 echo "==> Changing ownership"
 # be careful of what is chown'ed here. subdirs like mysql,redis etc are owned by the containers and will stop working if perms change
 chown -R "${USER}" /etc/cloudron
@@ -229,7 +206,7 @@ chown "${USER}:${USER}" "${BOX_DATA_DIR}/mail"
 chown "${USER}:${USER}" -R "${BOX_DATA_DIR}/mail/dkim" # this is owned by box currently since it generates the keys
 
 echo "==> Starting Cloudron"
-systemctl start box
+systemctl start cloudron.target
 
 sleep 2 # give systemd sometime to start the processes
 

setup/start/cloudron-firewall.sh

@@ -12,11 +12,6 @@ iptables -t filter -I CLOUDRON -m state --state RELATED,ESTABLISHED -j ACCEPT
 # ssh is allowed alternately on port 202
 iptables -A CLOUDRON -p tcp -m tcp -m multiport --dports 22,25,80,202,443,587,993,4190 -j ACCEPT
 
-# turn and stun service
-iptables -t filter -A CLOUDRON -p tcp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 3478,5349 -j ACCEPT
-iptables -t filter -A CLOUDRON -p udp -m multiport --dports 50000:51000 -j ACCEPT
-
 iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-request -j ACCEPT
 iptables -t filter -A CLOUDRON -p icmp --icmp-type echo-reply -j ACCEPT
 iptables -t filter -A CLOUDRON -p udp --sport 53 -j ACCEPT

setup/start/collectd/du.py

@@ -3,12 +3,10 @@ import collectd,os,subprocess,sys,re,time
 # https://www.programcreek.com/python/example/106897/collectd.register_read
 
 PATHS = [] # { name, dir, exclude }
-# there is a pattern in carbon/storage-schemas.conf which stores values every 12h for a year
 INTERVAL = 60 * 60 * 12 # twice a day. change values in docker-graphite if you change this
 
 def du(pathinfo):
-    # -B1 makes du print block sizes and not apparent sizes (to match df which also uses block sizes)
-    cmd = 'timeout 1800 du -DsB1 "{}"'.format(pathinfo['dir'])
+    cmd = 'timeout 1800 du -Dsb "{}"'.format(pathinfo['dir'])
     if pathinfo['exclude'] != '':
         cmd += ' --exclude "{}"'.format(pathinfo['exclude'])
 
@@ -28,7 +26,6 @@ def parseSize(size):
 
 def dockerSize():
     # use --format '{{json .}}' to dump the string. '{{if eq .Type "Images"}}{{.Size}}{{end}}' still creates newlines
-    # https://godoc.org/github.com/docker/go-units#HumanSize is used. so it's 1000 (KB) and not 1024 (KiB)
     cmd = 'timeout 1800 docker system df --format "{{.Size}}" | head -n1'
     try:
         size = subprocess.check_output(cmd, shell=True).strip().decode('utf-8')

setup/start/logrotate/platform

@@ -10,7 +10,6 @@
 /home/yellowtent/platformdata/logs/redis-*/*.log
 /home/yellowtent/platformdata/logs/crash/*.log
 /home/yellowtent/platformdata/logs/collectd/*.log
-/home/yellowtent/platformdata/logs/turn/*.log
 /home/yellowtent/platformdata/logs/updater/*.log {
     # only keep one rotated file, we currently do not send that over the api
     rotate 1
@@ -18,7 +17,6 @@
     missingok
     # we never compress so we can simply tail the files
     nocompress
-    # this truncates the original log file and not the rotated one
     copytruncate
 }
 

setup/start/nginx/nginx.conf

@@ -1,18 +1,11 @@
 user www-data;
 
-# detect based on available CPU cores
-worker_processes  auto;
-
-# this is 4096 by default. See /proc/<PID>/limits and /etc/security/limits.conf
-# usually twice the worker_connections (one for uptsream, one for downstream)
-# see also LimitNOFILE=16384 in systemd drop-in
-worker_rlimit_nofile 8192; 
+worker_processes  1;
 
 pid /run/nginx.pid;
 
 events {
-    # a single worker has these many simultaneous connections max
-    worker_connections  4096;
+    worker_connections  1024;
 }
 
 http {

setup/start/sudoers

@@ -50,12 +50,3 @@ yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartdocker.s
 Defaults!/home/yellowtent/box/src/scripts/restartunbound.sh env_keep="HOME BOX_ENV"
 yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/restartunbound.sh
 
-Defaults!/home/yellowtent/box/src/scripts/rmmailbox.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/rmmailbox.sh
-
-Defaults!/home/yellowtent/box/src/scripts/starttask.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD:SETENV: /home/yellowtent/box/src/scripts/starttask.sh
-
-Defaults!/home/yellowtent/box/src/scripts/stoptask.sh env_keep="HOME BOX_ENV"
-yellowtent ALL=(root) NOPASSWD: /home/yellowtent/box/src/scripts/stoptask.sh
-

setup/start/systemd/box.service

@@ -1,21 +1,20 @@
 [Unit]
 Description=Cloudron Admin
 OnFailure=crashnotifier@%n.service
+StopWhenUnneeded=true
 ; journald crashes result in a EPIPE in node. Cannot ignore it as it results in loss of logs.
 BindsTo=systemd-journald.service
 After=mysql.service nginx.service
 ; As cloudron-resize-fs is a one-shot, the Wants= automatically ensures that the service *finishes*
 Wants=cloudron-resize-fs.service
 
-[Install]
-WantedBy=multi-user.target
-
 [Service]
 Type=idle
 WorkingDirectory=/home/yellowtent/box
 Restart=always
-ExecStart=/home/yellowtent/box/box.js
-Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box:*,connect-lastmile,-box:ldap" "BOX_ENV=cloudron" "NODE_ENV=production"
+; Systemd does not append logs when logging to files, we spawn a shell first and exec to replace it after setting up the pipes
+ExecStart=/bin/sh -c 'echo "Logging to /home/yellowtent/platformdata/logs/box.log"; exec /usr/bin/node --max_old_space_size=150 /home/yellowtent/box/box.js >> /home/yellowtent/platformdata/logs/box.log 2>&1'
+Environment="HOME=/home/yellowtent" "USER=yellowtent" "DEBUG=box*,connect-lastmile" "BOX_ENV=cloudron" "NODE_ENV=production"
 ; kill apptask processes as well
 KillMode=control-group
 ; Do not kill this process on OOM. Children inherit this score. Do not set it to -1000 so that MemoryMax can keep working

setup/start/systemd/cloudron.target

@@ -0,0 +1,10 @@
+[Unit]
+Description=Cloudron Smartserver
+Documentation=https://cloudron.io/documentation.html
+StopWhenUnneeded=true
+Requires=box.service
+After=box.service
+# AllowIsolate=yes
+
+[Install]
+WantedBy=multi-user.target

setup/stop.sh

@@ -4,4 +4,4 @@ set -eu -o pipefail
 
 echo "Stopping cloudron"
 
-systemctl stop box
+systemctl stop cloudron.target

src/appdb.js

@@ -41,7 +41,7 @@ var assert = require('assert'),
 var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationState', 'apps.errorJson', 'apps.runState',
     'apps.health', 'apps.containerId', 'apps.manifestJson', 'apps.httpPort', 'subdomains.subdomain AS location', 'subdomains.domain',
     'apps.accessRestrictionJson', 'apps.memoryLimit', 'apps.cpuShares',
-    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson', 'apps.servicesConfigJson', 'apps.bindsJson',
+    'apps.label', 'apps.tagsJson', 'apps.taskId', 'apps.reverseProxyConfigJson',
     'apps.sso', 'apps.debugModeJson', 'apps.enableBackup',
     'apps.creationTime', 'apps.updateTime', 'apps.mailboxName', 'apps.mailboxDomain', 'apps.enableAutomaticUpdate',
     'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(',');
@@ -94,14 +94,6 @@ function postProcess(result) {
     result.debugMode = safe.JSON.parse(result.debugModeJson);
     delete result.debugModeJson;
 
-    assert(result.servicesConfigJson === null || typeof result.servicesConfigJson === 'string');
-    result.servicesConfig = safe.JSON.parse(result.servicesConfigJson) || {};
-    delete result.servicesConfigJson;
-
-    assert(result.bindsJson === null || typeof result.bindsJson === 'string');
-    result.binds = safe.JSON.parse(result.bindsJson) || {};
-    delete result.bindsJson;
-
     result.alternateDomains = result.alternateDomains || [];
     result.alternateDomains.forEach(function (d) {
         delete d.appId;
@@ -435,7 +427,7 @@ function updateWithConstraints(id, app, constraints, callback) {
 
     var fields = [ ], values = [ ];
     for (var p in app) {
-        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig' || p === 'servicesConfig' || p === 'binds') {
+        if (p === 'manifest' || p === 'tags' || p === 'accessRestriction' || p === 'debugMode' || p === 'error' || p === 'reverseProxyConfig') {
             fields.push(`${p}Json = ?`);
             values.push(JSON.stringify(app[p]));
         } else if (p !== 'portBindings' && p !== 'location' && p !== 'domain' && p !== 'alternateDomains' && p !== 'env') {

src/apphealthmonitor.js

@@ -73,6 +73,7 @@ function checkAppHealth(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
+        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
         return callback(null);
     }
 
@@ -102,8 +103,10 @@ function checkAppHealth(app, callback) {
             .timeout(HEALTHCHECK_INTERVAL)
             .end(function (error, res) {
                 if (error && !error.response) {
+                    debugApp(app, 'not alive (network error): %s', error.message);
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                 } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
+                    debugApp(app, 'not alive : %s', error || res.status);
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                 } else {
                     setHealth(app, apps.HEALTH_HEALTHY, callback);
@@ -177,14 +180,18 @@ function processDockerEvents(intervalSecs, callback) {
 function processApp(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    apps.getAll(function (error, allApps) {
+    apps.getAll(function (error, result) {
         if (error) return callback(error);
 
-        async.each(allApps, checkAppHealth, function (error) {
-            const alive = allApps
-                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
+        async.each(result, checkAppHealth, function (error) {
+            if (error) console.error(error);
 
-            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead.` + (error ? ` ${error.reason}` : ''));
+            const alive = result
+                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
+                .map(a => a.fqdn)
+                .join(', ');
+
+            debug('apps alive: [%s]', alive);
 
             callback(null);
         });
@@ -199,7 +206,7 @@ function run(intervalSecs, callback) {
         processApp, // this is first because docker.getEvents seems to get 'stuck' sometimes
         processDockerEvents.bind(null, intervalSecs)
     ], function (error) {
-        if (error) debug(`run: could not check app health. ${error.message}`);
+        if (error) debug(error);
 
         callback();
     });

src/appstore.js

@@ -11,6 +11,7 @@ exports = module.exports = {
     trackFinishedSetup: trackFinishedSetup,
 
     registerWithLoginCredentials: registerWithLoginCredentials,
+    registerWithLicense: registerWithLicense,
 
     purchaseApp: purchaseApp,
     unpurchaseApp: unpurchaseApp,
@@ -19,6 +20,8 @@ exports = module.exports = {
     getSubscription: getSubscription,
     isFreePlan: isFreePlan,
 
+    sendAliveStatus: sendAliveStatus,
+
     getAppUpdate: getAppUpdate,
     getBoxUpdate: getBoxUpdate,
 
@@ -31,26 +34,32 @@ var apps = require('./apps.js'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:appstore'),
+    domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
+    groups = require('./groups.js'),
+    mail = require('./mail.js'),
+    os = require('os'),
     paths = require('./paths.js'),
     safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
     superagent = require('superagent'),
+    users = require('./users.js'),
     util = require('util');
 
+const NOOP_CALLBACK = function (error) { if (error) debug(error); };
+
 // These are the default options and will be adjusted once a subscription state is obtained
 // Keep in sync with appstore/routes/cloudrons.js
 let gFeatures = {
-    userMaxCount: 5,
-    domainMaxCount: 1,
-    externalLdap: false,
-    privateDockerRegistry: false,
-    branding: false,
-    support: false,
-    directoryConfig: false,
-    mailboxMaxCount: 5,
-    emailPremium: false
+    userMaxCount: null,
+    externalLdap: true,
+    eventLog: true,
+    privateDockerRegistry: true,
+    branding: true,
+    userManager: true,
+    multiAdmin: true,
+    support: true
 };
 
 // attempt to load feature cache in case appstore would be down
@@ -118,7 +127,7 @@ function registerUser(email, password, callback) {
     const url = settings.apiServerOrigin() + '/api/v1/register_user';
     superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS, error.message));
+        if (result.statusCode === 409) return callback(new BoxError(BoxError.ALREADY_EXISTS));
         if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `register status code: ${result.statusCode}`));
 
         callback(null);
@@ -226,8 +235,112 @@ function unpurchaseApp(appId, data, callback) {
     });
 }
 
-function getBoxUpdate(options, callback) {
-    assert.strictEqual(typeof options, 'object');
+function sendAliveStatus(callback) {
+    callback = callback || NOOP_CALLBACK;
+
+    let allSettings, allDomains, mailDomains, loginEvents, userCount, groupCount;
+
+    async.series([
+        function (callback) {
+            settings.getAll(function (error, result) {
+                if (error) return callback(error);
+                allSettings = result;
+                callback();
+            });
+        },
+        function (callback) {
+            domains.getAll(function (error, result) {
+                if (error) return callback(error);
+                allDomains = result;
+                callback();
+            });
+        },
+        function (callback) {
+            mail.getDomains(function (error, result) {
+                if (error) return callback(error);
+                mailDomains = result;
+                callback();
+            });
+        },
+        function (callback) {
+            eventlog.getAllPaged([ eventlog.ACTION_USER_LOGIN ], null, 1, 1, function (error, result) {
+                if (error) return callback(error);
+                loginEvents = result;
+                callback();
+            });
+        },
+        function (callback) {
+            users.count(function (error, result) {
+                if (error) return callback(error);
+                userCount = result;
+                callback();
+            });
+        },
+        function (callback) {
+            groups.count(function (error, result) {
+                if (error) return callback(error);
+                groupCount = result;
+                callback();
+            });
+        }
+    ], function (error) {
+        if (error) return callback(error);
+
+        var backendSettings = {
+            backupConfig: {
+                provider: allSettings[settings.BACKUP_CONFIG_KEY].provider,
+                hardlinks: !allSettings[settings.BACKUP_CONFIG_KEY].noHardlinks
+            },
+            domainConfig: {
+                count: allDomains.length,
+                domains: Array.from(new Set(allDomains.map(function (d) { return { domain: d.domain, provider: d.provider }; })))
+            },
+            mailConfig: {
+                outboundCount: mailDomains.length,
+                inboundCount: mailDomains.filter(function (d) { return d.enabled; }).length,
+                catchAllCount: mailDomains.filter(function (d) { return d.catchAll.length !== 0; }).length,
+                relayProviders: Array.from(new Set(mailDomains.map(function (d) { return d.relay.provider; })))
+            },
+            userCount: userCount,
+            groupCount: groupCount,
+            appAutoupdatePattern: allSettings[settings.APP_AUTOUPDATE_PATTERN_KEY],
+            boxAutoupdatePattern: allSettings[settings.BOX_AUTOUPDATE_PATTERN_KEY],
+            timeZone: allSettings[settings.TIME_ZONE_KEY],
+            sysinfoProvider: allSettings[settings.SYSINFO_CONFIG_KEY].provider
+        };
+
+        var data = {
+            version: constants.VERSION,
+            adminFqdn: settings.adminFqdn(),
+            provider: settings.provider(),
+            backendSettings: backendSettings,
+            machine: {
+                cpus: os.cpus(),
+                totalmem: os.totalmem()
+            },
+            events: {
+                lastLogin: loginEvents[0] ? (new Date(loginEvents[0].creationTime).getTime()) : 0
+            }
+        };
+
+        getCloudronToken(function (error, token) {
+            if (error) return callback(error);
+
+            const url = `${settings.apiServerOrigin()}/api/v1/alive`;
+            superagent.post(url).send(data).query({ accessToken: token }).timeout(30 * 1000).end(function (error, result) {
+                if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
+                if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+                if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
+                if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Sending alive status failed. %s %j', result.status, result.body)));
+
+                callback(null);
+            });
+        });
+    });
+}
+
+function getBoxUpdate(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     getCloudronToken(function (error, token) {
@@ -235,13 +348,7 @@ function getBoxUpdate(options, callback) {
 
         const url = `${settings.apiServerOrigin()}/api/v1/boxupdate`;
 
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            automatic: options.automatic
-        };
-
-        superagent.get(url).query(query).timeout(10 * 1000).end(function (error, result) {
+        superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION }).timeout(10 * 1000).end(function (error, result) {
             if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
             if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
@@ -267,24 +374,16 @@ function getBoxUpdate(options, callback) {
     });
 }
 
-function getAppUpdate(app, options, callback) {
+function getAppUpdate(app, callback) {
     assert.strictEqual(typeof app, 'object');
-    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     getCloudronToken(function (error, token) {
         if (error) return callback(error);
 
         const url = `${settings.apiServerOrigin()}/api/v1/appupdate`;
-        const query = {
-            accessToken: token,
-            boxVersion: constants.VERSION,
-            appId: app.appStoreId,
-            appVersion: app.manifest.version,
-            automatic: options.automatic
-        };
 
-        superagent.get(url).query(query).timeout(10 * 1000).end(function (error, result) {
+        superagent.get(url).query({ accessToken: token, boxVersion: constants.VERSION, appId: app.appStoreId, appVersion: app.manifest.version }).timeout(10 * 1000).end(function (error, result) {
             if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error));
             if (result.statusCode === 401) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
             if (result.statusCode === 422) return callback(new BoxError(BoxError.LICENSE_ERROR, result.body.message));
@@ -302,9 +401,7 @@ function getAppUpdate(app, options, callback) {
                 return callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('Malformed update: %s %s', result.statusCode, result.text)));
             }
 
-            updateInfo.unstable = !!updateInfo.unstable;
-
-            // { id, creationDate, manifest, unstable }
+            // { id, creationDate, manifest }
             callback(null, updateInfo);
         });
     });
@@ -318,7 +415,7 @@ function registerCloudron(data, callback) {
 
     superagent.post(url).send(data).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
-        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${result.statusCode} ${error.message}`));
+        if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, `Unable to register cloudron: ${error.message}`));
 
         // cloudronId, token, licenseKey
         if (!result.body.cloudronId) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Invalid response - no cloudron id'));
@@ -341,16 +438,18 @@ function registerCloudron(data, callback) {
 
 // This works without a Cloudron token as this Cloudron was not yet registered
 let gBeginSetupAlreadyTracked = false;
-function trackBeginSetup() {
+function trackBeginSetup(provider) {
+    assert.strictEqual(typeof provider, 'string');
+
     // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
     if (gBeginSetupAlreadyTracked) return;
     gBeginSetupAlreadyTracked = true;
 
     const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
 
-    superagent.post(url).send({}).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackBeginSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackBeginSetup: ${result.statusCode} ${error.message}`);
+    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
     });
 }
 
@@ -361,8 +460,23 @@ function trackFinishedSetup(domain) {
     const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_finished`;
 
     superagent.post(url).send({ domain }).timeout(30 * 1000).end(function (error, result) {
-        if (error && !error.response) return debug(`trackFinishedSetup: ${error.message}`);
-        if (result.statusCode !== 200) return debug(`trackFinishedSetup: ${result.statusCode} ${error.message}`);
+        if (error && !error.response) return console.error(error.message);
+        if (result.statusCode !== 200) return console.error(error.message);
+    });
+}
+
+function registerWithLicense(license, domain, callback) {
+    assert.strictEqual(typeof license, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    getCloudronToken(function (error, token) {
+        if (token) return callback(new BoxError(BoxError.CONFLICT));
+
+        const provider = settings.provider();
+        const version = constants.VERSION;
+
+        registerCloudron({ license, domain, provider, version }, callback);
     });
 }
 
@@ -377,7 +491,7 @@ function registerWithLoginCredentials(options, callback) {
     }
 
     getCloudronToken(function (error, token) {
-        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
+        if (token) return callback(new BoxError(BoxError.CONFLICT));
 
         maybeSignup(function (error) {
             if (error) return callback(error);
@@ -385,7 +499,7 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION, purpose: options.purpose || '' }, callback);
             });
         });
     });
@@ -410,7 +524,7 @@ function createTicket(info, auditSource, callback) {
         if (error) return callback(error);
 
         collectAppInfoIfNeeded(function (error, result) {
-            if (error) return callback(error);
+            if (error) console.error('Unable to get app info', error);
             if (result) info.app = result;
 
             let url = settings.apiServerOrigin() + '/api/v1/ticket';

src/apptask.js

@@ -17,6 +17,8 @@ exports = module.exports = {
     _waitForDnsPropagation: waitForDnsPropagation
 };
 
+require('supererror')({ splatchError: true });
+
 var addons = require('./addons.js'),
     appdb = require('./appdb.js'),
     apps = require('./apps.js'),
@@ -35,6 +37,7 @@ var addons = require('./addons.js'),
     eventlog = require('./eventlog.js'),
     fs = require('fs'),
     manifestFormat = require('cloudron-manifestformat'),
+    mkdirp = require('mkdirp'),
     net = require('net'),
     os = require('os'),
     path = require('path'),
@@ -43,7 +46,6 @@ var addons = require('./addons.js'),
     rimraf = require('rimraf'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
-    sftp = require('./sftp.js'),
     shell = require('./shell.js'),
     superagent = require('superagent'),
     sysinfo = require('./sysinfo.js'),
@@ -174,7 +176,7 @@ function createAppDir(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const appDir = path.join(paths.APPS_DATA_DIR, app.id);
-    fs.mkdir(appDir, { recursive: true }, function (error) {
+    mkdirp(appDir, function (error) {
         if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error creating directory: ${error.message}`, { appDir }));
 
         callback(null);
@@ -578,9 +580,6 @@ function install(app, args, progressCallback, callback) {
 
         startApp.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Configuring file manager' }),
-        sftp.rebuild.bind(null, {}),
-
         progressCallback.bind(null, { percent: 85, message: 'Waiting for DNS propagation' }),
         exports._waitForDnsPropagation.bind(null, app),
 
@@ -742,12 +741,7 @@ function migrateDataDir(app, args, progressCallback, callback) {
             debugApp(app, 'error migrating data dir : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
-
-        // We do this after the app has the new data commited to the database
-        sftp.rebuild({}, function (error) {
-            if (error) debug('migrateDataDir: failed to rebuild sftp addon:', error);
-            callback();
-        });
+        callback(null);
     });
 }
 
@@ -782,9 +776,6 @@ function configure(app, args, progressCallback, callback) {
 
         startApp.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Configuring file manager' }),
-        sftp.rebuild.bind(null, {}),
-
         progressCallback.bind(null, { percent: 90, message: 'Configuring reverse proxy' }),
         configureReverseProxy.bind(null, app),
 
@@ -795,8 +786,7 @@ function configure(app, args, progressCallback, callback) {
             debugApp(app, 'error reconfiguring : %s', error);
             return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
         }
-
-        callback();
+        callback(null);
     });
 }
 
@@ -863,7 +853,7 @@ function update(app, args, progressCallback, callback) {
                 if (newTcpPorts[portName] || newUdpPorts[portName]) return callback(null); // port still in use
 
                 appdb.delPortBinding(currentPorts[portName], apps.PORT_TYPE_TCP, function (error) {
-                    if (error && error.reason === BoxError.NOT_FOUND) debug('update: portbinding does not exist in database', error);
+                    if (error && error.reason === BoxError.NOT_FOUND) console.error('Portbinding does not exist in database.');
                     else if (error) return next(error);
 
                     // also delete from app object for further processing (the db is updated in the next step)
@@ -879,17 +869,14 @@ function update(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 45, message: 'Downloading icon' }),
         downloadIcon.bind(null, app),
 
-        progressCallback.bind(null, { percent: 60, message: 'Updating addons' }),
+        progressCallback.bind(null, { percent: 70, message: 'Updating addons' }),
         addons.setupAddons.bind(null, app, updateConfig.manifest.addons),
 
-        progressCallback.bind(null, { percent: 70, message: 'Creating container' }),
+        progressCallback.bind(null, { percent: 80, message: 'Creating container' }),
         createContainer.bind(null, app),
 
         startApp.bind(null, app),
 
-        progressCallback.bind(null, { percent: 80, message: 'Configuring file manager' }),
-        sftp.rebuild.bind(null, {}),
-
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null, updateTime: new Date() })
     ], function seriesDone(error) {
@@ -912,10 +899,7 @@ function start(app, args, progressCallback, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     async.series([
-        progressCallback.bind(null, { percent: 10, message: 'Starting app services' }),
-        addons.startAppServices.bind(null, app),
-
-        progressCallback.bind(null, { percent: 35, message: 'Starting container' }),
+        progressCallback.bind(null, { percent: 20, message: 'Starting container' }),
         docker.startContainer.bind(null, app.id),
 
         // stopped apps do not renew certs. currently, we don't do DNS to not overwrite existing user settings
@@ -943,9 +927,6 @@ function stop(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 20, message: 'Stopping container' }),
         docker.stopContainers.bind(null, app.id),
 
-        progressCallback.bind(null, { percent: 50, message: 'Stopping app services' }),
-        addons.stopAppServices.bind(null, app),
-
         progressCallback.bind(null, { percent: 100, message: 'Done' }),
         updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
     ], function seriesDone(error) {
@@ -992,22 +973,16 @@ function uninstall(app, args, progressCallback, callback) {
         progressCallback.bind(null, { percent: 30, message: 'Teardown addons' }),
         addons.teardownAddons.bind(null, app, app.manifest.addons),
 
-        progressCallback.bind(null, { percent: 40, message: 'Cleanup file manager' }),
-        function (callback) {
-            if (!app.dataDir) return callback();
-            sftp.rebuild({ ignoredApps: [ app.id ] }, callback);
-        },
-
-        progressCallback.bind(null, { percent: 50, message: 'Deleting app data directory' }),
+        progressCallback.bind(null, { percent: 40, message: 'Deleting app data directory' }),
         deleteAppDir.bind(null, app, { removeDirectory: true }),
 
-        progressCallback.bind(null, { percent: 60, message: 'Deleting image' }),
+        progressCallback.bind(null, { percent: 50, message: 'Deleting image' }),
         docker.deleteImage.bind(null, app.manifest),
 
-        progressCallback.bind(null, { percent: 70, message: 'Unregistering domains' }),
+        progressCallback.bind(null, { percent: 60, message: 'Unregistering domains' }),
         unregisterSubdomains.bind(null, app, [ { subdomain: app.location, domain: app.domain } ].concat(app.alternateDomains)),
 
-        progressCallback.bind(null, { percent: 80, message: 'Cleanup icon' }),
+        progressCallback.bind(null, { percent: 70, message: 'Cleanup icon' }),
         removeIcon.bind(null, app),
 
         progressCallback.bind(null, { percent: 90, message: 'Cleanup logs' }),
@@ -1063,8 +1038,6 @@ function run(appId, args, progressCallback, callback) {
             return stop(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_RESTART:
             return restart(app, args, progressCallback, callback);
-        case apps.ISTATE_INSTALLED: // can only happen when we have a bug in our code while testing/development
-            return updateApp(app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null }, callback);
         default:
             debugApp(app, 'apptask launched with invalid command');
             return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));

src/apptaskmanager.js

@@ -68,8 +68,7 @@ function scheduleTask(appId, taskId, callback) {
 
     if (!fs.existsSync(path.dirname(logFile))) safe.fs.mkdirSync(path.dirname(logFile)); // ensure directory
 
-    // TODO: set memory limit for app backup task
-    tasks.startTask(taskId, { logFile, timeout: 20 * 60 * 60 * 1000 /* 20 hours */, nice: 15 }, function (error, result) {
+    tasks.startTask(taskId, { logFile, timeout: 20 * 60 * 60 * 1000 /* 20 hours */ }, function (error, result) {
         callback(error, result);
 
         delete gActiveTasks[appId];

src/backupdb.js

@@ -6,20 +6,27 @@ var assert = require('assert'),
     safe = require('safetydance'),
     util = require('util');
 
-var BACKUPS_FIELDS = [ 'id', 'identifier', 'creationTime', 'packageVersion', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs', 'encryptionVersion' ];
+var BACKUPS_FIELDS = [ 'id', 'creationTime', 'version', 'type', 'dependsOn', 'state', 'manifestJson', 'format', 'preserveSecs' ];
 
 exports = module.exports = {
-    add,
+    add: add,
 
-    getByTypePaged,
-    getByIdentifierPaged,
-    getByIdentifierAndStatePaged,
+    getByTypeAndStatePaged: getByTypeAndStatePaged,
+    getByTypePaged: getByTypePaged,
 
-    get,
-    del,
-    update,
+    get: get,
+    del: del,
+    update: update,
+    getByAppIdPaged: getByAppIdPaged,
 
-    _clear: clear
+    _clear: clear,
+
+    BACKUP_TYPE_APP: 'app',
+    BACKUP_TYPE_BOX: 'box',
+
+    BACKUP_STATE_NORMAL: 'normal', // should rename to created to avoid listing in UI?
+    BACKUP_STATE_CREATING: 'creating',
+    BACKUP_STATE_ERROR: 'error'
 };
 
 function postProcess(result) {
@@ -31,15 +38,15 @@ function postProcess(result) {
     delete result.manifestJson;
 }
 
-function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
+function getByTypeAndStatePaged(type, state, page, perPage, callback) {
+    assert(type === exports.BACKUP_TYPE_APP || type === exports.BACKUP_TYPE_BOX);
     assert.strictEqual(typeof state, 'string');
     assert(typeof page === 'number' && page > 0);
     assert(typeof perPage === 'number' && perPage > 0);
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, state, (page-1)*perPage, perPage ], function (error, results) {
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ type, state, (page-1)*perPage, perPage ], function (error, results) {
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
             results.forEach(function (result) { postProcess(result); });
@@ -49,7 +56,7 @@ function getByIdentifierAndStatePaged(identifier, state, page, perPage, callback
 }
 
 function getByTypePaged(type, page, perPage, callback) {
-    assert.strictEqual(typeof type, 'string');
+    assert(type === exports.BACKUP_TYPE_APP || type === exports.BACKUP_TYPE_BOX);
     assert(typeof page === 'number' && page > 0);
     assert(typeof perPage === 'number' && perPage > 0);
     assert.strictEqual(typeof callback, 'function');
@@ -64,14 +71,15 @@ function getByTypePaged(type, page, perPage, callback) {
         });
 }
 
-function getByIdentifierPaged(identifier, page, perPage, callback) {
-    assert.strictEqual(typeof identifier, 'string');
+function getByAppIdPaged(page, perPage, appId, callback) {
     assert(typeof page === 'number' && page > 0);
     assert(typeof perPage === 'number' && perPage > 0);
+    assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE identifier = ? ORDER BY creationTime DESC LIMIT ?,?',
-        [ identifier, (page-1)*perPage, perPage ], function (error, results) {
+    // box versions (0.93.x and below) used to use appbackup_ prefix
+    database.query('SELECT ' + BACKUPS_FIELDS + ' FROM backups WHERE type = ? AND state = ? AND id LIKE ? ORDER BY creationTime DESC LIMIT ?,?',
+        [ exports.BACKUP_TYPE_APP, exports.BACKUP_STATE_NORMAL, '%app%\\_' + appId + '\\_%', (page-1)*perPage, perPage ], function (error, results) {
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
             results.forEach(function (result) { postProcess(result); });
@@ -98,11 +106,8 @@ function get(id, callback) {
 function add(id, data, callback) {
     assert(data && typeof data === 'object');
     assert.strictEqual(typeof id, 'string');
-    assert(data.encryptionVersion === null || typeof data.encryptionVersion === 'number');
-    assert.strictEqual(typeof data.packageVersion, 'string');
-    assert.strictEqual(typeof data.type, 'string');
-    assert.strictEqual(typeof data.identifier, 'string');
-    assert.strictEqual(typeof data.state, 'string');
+    assert.strictEqual(typeof data.version, 'string');
+    assert(data.type === exports.BACKUP_TYPE_APP || data.type === exports.BACKUP_TYPE_BOX);
     assert(util.isArray(data.dependsOn));
     assert.strictEqual(typeof data.manifest, 'object');
     assert.strictEqual(typeof data.format, 'string');
@@ -111,8 +116,8 @@ function add(id, data, callback) {
     var creationTime = data.creationTime || new Date(); // allow tests to set the time
     var manifestJson = JSON.stringify(data.manifest);
 
-    database.query('INSERT INTO backups (id, identifier, encryptionVersion, packageVersion, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
-        [ id, data.identifier, data.encryptionVersion, data.packageVersion, data.type, creationTime, data.state, data.dependsOn.join(','), manifestJson, data.format ],
+    database.query('INSERT INTO backups (id, version, type, creationTime, state, dependsOn, manifestJson, format) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+        [ id, data.version, data.type, creationTime, exports.BACKUP_STATE_NORMAL, data.dependsOn.join(','), manifestJson, data.format ],
         function (error) {
             if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS));
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));

src/cert/acme2.js

@@ -332,7 +332,7 @@ Acme2.prototype.createKeyAndCsr = function (hostname, callback) {
         // in some old releases, csr file was corrupt. so always regenerate it
         debug('createKeyAndCsr: reuse the key for renewal at %s', privateKeyFile);
     } else {
-        var key = safe.child_process.execSync('openssl ecparam -genkey -name secp384r1'); // openssl ecparam -list_curves
+        var key = safe.child_process.execSync('openssl genrsa 4096');
         if (!key) return callback(new BoxError(BoxError.OPENSSL_ERROR, safe.error));
         if (!safe.fs.writeFileSync(privateKeyFile, key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error));
 

src/cert/caas.js

@@ -0,0 +1,22 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'caas'
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:cert/caas.js');
+
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: using fallback certificate', hostname);
+
+    return callback(null, '', '');
+}

src/cert/fallback.js

@@ -0,0 +1,22 @@
+'use strict';
+
+exports = module.exports = {
+    getCertificate: getCertificate,
+
+    // testing
+    _name: 'fallback'
+};
+
+var assert = require('assert'),
+    debug = require('debug')('box:cert/fallback.js');
+
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('getCertificate: using fallback certificate', hostname);
+
+    return callback(null, '', '');
+}

src/cert/interface.js

@@ -0,0 +1,24 @@
+'use strict';
+
+// -------------------------------------------
+//  This file just describes the interface
+//
+//  New backends can start from here
+// -------------------------------------------
+
+exports = module.exports = {
+    getCertificate: getCertificate
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js');
+
+function getCertificate(hostname, domain, options, callback) {
+    assert.strictEqual(typeof hostname, 'string');
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof options, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    return callback(new BoxError(BoxError.NOT_IMPLEMENTED, 'getCertificate is not implemented'));
+}
+

src/cloudron.js

@@ -18,11 +18,10 @@ exports = module.exports = {
 
     setupDashboard: setupDashboard,
 
-    runSystemChecks: runSystemChecks
+    runSystemChecks: runSystemChecks,
 };
 
-var addons = require('./addons.js'),
-    apps = require('./apps.js'),
+var apps = require('./apps.js'),
     appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
@@ -66,7 +65,7 @@ function uninitialize(callback) {
 
     async.series([
         cron.stopJobs,
-        platform.stopAllTasks
+        platform.stop
     ], callback);
 }
 
@@ -78,13 +77,7 @@ function onActivated(callback) {
     // 2. the restore code path can run without sudo (since mail/ is non-root)
     async.series([
         platform.start,
-        cron.startJobs,
-        function checkBackupConfiguration(callback) {
-            backups.checkConfiguration(function (error, message) {
-                if (error) return callback(error);
-                notifications.alert(notifications.ALERT_BACKUP_CONFIG, 'Backup configuration is unsafe', message, callback);
-            });
-        }
+        cron.startJobs
     ], callback);
 }
 
@@ -109,15 +102,12 @@ function notifyUpdate(callback) {
 
 // each of these tasks can fail. we will add some routes to fix/re-run them
 function runStartupTasks() {
-    // stop all the systemd tasks
-    platform.stopAllTasks(NOOP_CALLBACK);
-
     // configure nginx to be reachable by IP
     reverseProxy.writeDefaultConfig(NOOP_CALLBACK);
 
     // this configures collectd to collect backup storage metrics if filesystem is used. This is also triggerd when the settings change with the rest api
     settings.getBackupConfig(function (error, backupConfig) {
-        if (error) return debug('runStartupTasks: failed to get backup config.', error);
+        if (error) return console.error('Failed to read backup config.', error);
         backups.configureCollectd(backupConfig, NOOP_CALLBACK);
     });
 
@@ -148,18 +138,17 @@ function getConfig(callback) {
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
             isDemo: settings.isDemo(),
+            provider: settings.provider(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
             footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
-            features: appstore.getFeatures(),
-            profileLocked: allSettings[settings.DIRECTORY_CONFIG_KEY].lockUserProfiles,
-            mandatory2FA: allSettings[settings.DIRECTORY_CONFIG_KEY].mandatory2FA
+            features: appstore.getFeatures()
         });
     });
 }
 
 function reboot(callback) {
     notifications.alert(notifications.ALERT_REBOOT, 'Reboot Required', '', function (error) {
-        if (error) debug('reboot: failed to clear reboot notification.', error);
+        if (error) console.error('Failed to clear reboot notification.', error);
 
         shell.sudo('reboot', [ REBOOT_CMD ], {}, callback);
     });
@@ -177,11 +166,24 @@ function runSystemChecks(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     async.parallel([
+        checkBackupConfiguration,
         checkMailStatus,
         checkRebootRequired
     ], callback);
 }
 
+function checkBackupConfiguration(callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('checking backup configuration');
+
+    backups.checkConfiguration(function (error, message) {
+        if (error) return callback(error);
+
+        notifications.alert(notifications.ALERT_BACKUP_CONFIG, 'Backup configuration is unsafe', message, callback);
+    });
+}
+
 function checkMailStatus(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -324,7 +326,6 @@ function setDashboardAndMailDomain(domain, auditSource, callback) {
         if (error) return callback(error);
 
         mail.onMailFqdnChanged(NOOP_CALLBACK); // this will update dns and re-configure mail server
-        addons.restartService('turn', NOOP_CALLBACK); // to update the realm variable
 
         callback(null);
     });

src/constants.js

@@ -37,11 +37,10 @@ exports = module.exports = {
     DEFAULT_MEMORY_LIMIT: (256 * 1024 * 1024), // see also client.js
 
     DEMO_USERNAME: 'cloudron',
-    DEMO_BLACKLISTED_APPS: [ 'com.github.cloudtorrent' ],
 
     AUTOUPDATE_PATTERN_NEVER: 'never',
 
-    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8), // also used in dashboard client.js
+    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),
 
     CLOUDRON: CLOUDRON,
     TEST: TEST,
@@ -50,6 +49,6 @@ exports = module.exports = {
 
     FOOTER: '&copy; 2020  &nbsp;  [Cloudron](https://cloudron.io) &nbsp; &nbsp; &nbsp;  [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)',
 
-    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '5.1.1-test'
+    VERSION: process.env.BOX_ENV === 'cloudron' ? fs.readFileSync(path.join(__dirname, '../VERSION'), 'utf8').trim() : '4.2.0-test'
 };
 

src/cron.js

@@ -1,25 +1,16 @@
 'use strict';
 
-// IMPORTANT: These patterns are together because they spin tasks which acquire a lock
-// If the patterns overlap all the time, then the task may not ever get a chance to run!
-// If you change this change dashboard patterns in settings.html
-const DEFAULT_CLEANUP_BACKUPS_PATTERN = '00 30 1,3,5,23 * * *',
-    DEFAULT_BOX_AUTOUPDATE_PATTERN = '00 00 1,3,5,23 * * *',
-    DEFAULT_APP_AUTOUPDATE_PATTERN = '00 15 1,3,5,23 * * *';
-
 exports = module.exports = {
-    startJobs,
+    startJobs: startJobs,
 
-    stopJobs,
+    stopJobs: stopJobs,
 
-    handleSettingsChanged,
-
-    DEFAULT_BOX_AUTOUPDATE_PATTERN,
-    DEFAULT_APP_AUTOUPDATE_PATTERN
+    handleSettingsChanged: handleSettingsChanged
 };
 
 var appHealthMonitor = require('./apphealthmonitor.js'),
     apps = require('./apps.js'),
+    appstore = require('./appstore.js'),
     assert = require('assert'),
     async = require('async'),
     auditSource = require('./auditsource.js'),
@@ -38,6 +29,7 @@ var appHealthMonitor = require('./apphealthmonitor.js'),
     updateChecker = require('./updatechecker.js');
 
 var gJobs = {
+    alive: null, // send periodic stats
     appAutoUpdater: null,
     boxAutoUpdater: null,
     appUpdateChecker: null,
@@ -69,8 +61,14 @@ function startJobs(callback) {
     assert.strictEqual(typeof callback, 'function');
 
     const randomMinute = Math.floor(60*Math.random());
+    gJobs.alive = new CronJob({
+        cronTime: '00 ' + randomMinute + ' * * * *', // every hour on a random minute
+        onTick: appstore.sendAliveStatus,
+        start: true
+    });
+
     gJobs.systemChecks = new CronJob({
-        cronTime: '00 30 2 * * *', // once a day. if you change this interval, change the notification messages with correct duration
+        cronTime: '00 30 * * * *', // every 30 minutes. if you change this interval, change the notification messages with correct duration
         onTick: () => cloudron.runSystemChecks(NOOP_CALLBACK),
         start: true
     });
@@ -82,15 +80,14 @@ function startJobs(callback) {
     });
 
     gJobs.boxUpdateCheckerJob = new CronJob({
-        cronTime: '00 ' + randomMinute + ' 1,3,5,21,23 * * *', // 5 times
-        onTick: () => updateChecker.checkBoxUpdates({ automatic: true }, NOOP_CALLBACK),
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkBoxUpdates(NOOP_CALLBACK),
         start: true
     });
 
-    // this is run separately from the update itself so that the user can disable automatic updates but can still get a notification
     gJobs.appUpdateChecker = new CronJob({
-        cronTime: '00 ' + randomMinute + ' 2,4,6,20,22 * * *', // 5 times
-        onTick: () => updateChecker.checkAppUpdates({ automatic: true }, NOOP_CALLBACK),
+        cronTime: '00 ' + randomMinute + ' * * * *', // once an hour
+        onTick: () => updateChecker.checkAppUpdates(NOOP_CALLBACK),
         start: true
     });
 
@@ -101,7 +98,7 @@ function startJobs(callback) {
     });
 
     gJobs.cleanupBackups = new CronJob({
-        cronTime: DEFAULT_CLEANUP_BACKUPS_PATTERN,
+        cronTime: '00 45 1,3,5,23 * * *', // every 6 hours. try not to overlap with ensureBackup job
         onTick: backups.startCleanupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true
     });
@@ -175,13 +172,19 @@ function backupConfigChanged(value, tz) {
     assert.strictEqual(typeof value, 'object');
     assert.strictEqual(typeof tz, 'string');
 
-    debug(`backupConfigChanged: schedule ${value.schedulePattern} (${tz})`);
+    debug(`backupConfigChanged: interval ${value.intervalSecs} (${tz})`);
 
     if (gJobs.backup) gJobs.backup.stop();
+    let pattern;
+    if (value.intervalSecs <= 6 * 60 * 60) {
+        pattern = '00 00 1,7,13,19 * * *'; // no option but to backup in the middle of the day
+    } else {
+        pattern = '00 00 1,3,5,23 * * *'; // avoid middle of the day backups
+    }
 
     gJobs.backup = new CronJob({
-        cronTime: value.schedulePattern,
-        onTick: backups.startBackupTask.bind(null, auditSource.CRON, NOOP_CALLBACK),
+        cronTime: pattern,
+        onTick: backups.ensureBackup.bind(null, auditSource.CRON, NOOP_CALLBACK),
         start: true,
         timeZone: tz
     });

src/database.js

@@ -17,12 +17,12 @@ var assert = require('assert'),
     BoxError = require('./boxerror.js'),
     child_process = require('child_process'),
     constants = require('./constants.js'),
-    debug = require('debug')('box:database'),
     mysql = require('mysql'),
     once = require('once'),
     util = require('util');
 
-var gConnectionPool = null;
+var gConnectionPool = null,
+    gDefaultConnection = null;
 
 const gDatabase = {
     hostname: '127.0.0.1',
@@ -42,37 +42,59 @@ function initialize(callback) {
         gDatabase.hostname = require('child_process').execSync('docker inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" mysql-server').toString().trim();
     }
 
-    // https://github.com/mysqljs/mysql#pool-options
     gConnectionPool  = mysql.createPool({
-        connectionLimit: 5,
+        connectionLimit: 5, // this has to be > 1 since we store one connection as 'default'. the rest for transactions
         host: gDatabase.hostname,
         user: gDatabase.username,
         password: gDatabase.password,
         port: gDatabase.port,
         database: gDatabase.name,
         multipleStatements: false,
-        waitForConnections: true, // getConnection() will wait until a connection is avaiable
         ssl: false,
         timezone: 'Z' // mysql follows the SYSTEM timezone. on Cloudron, this is UTC
     });
 
     gConnectionPool.on('connection', function (connection) {
-        // connection objects are re-used. so we have to attach to the event here (once) to prevent crash
-        // note the pool also has an 'acquire' event but that is called whenever we do a getConnection()
-        connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`));
-
         connection.query('USE ' + gDatabase.name);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
     });
 
-    callback(null);
+    reconnect(callback);
 }
 
 function uninitialize(callback) {
-    if (!gConnectionPool) return callback(null);
+    if (gConnectionPool) {
+        gConnectionPool.end(callback);
+        gConnectionPool = null;
+    } else {
+        callback(null);
+    }
+}
 
-    gConnectionPool.end(callback);
-    gConnectionPool = null;
+function reconnect(callback) {
+    callback = callback ? once(callback) : function () {};
+
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) {
+            console.error('Unable to reestablish connection to database. Try again in a bit.', error.message);
+            return setTimeout(reconnect.bind(null, callback), 1000);
+        }
+
+        connection.on('error', function (error) {
+            // by design, we catch all normal errors by providing callbacks.
+            // this function should be invoked only when we have no callbacks pending and we have a fatal error
+            assert(error.fatal, 'Non-fatal error on connection object');
+
+            console.error('Unhandled mysql connection error.', error);
+
+            // This is most likely an issue an can cause double callbacks from reconnect()
+            setTimeout(reconnect.bind(null, callback), 1000);
+        });
+
+        gDefaultConnection = connection;
+
+        callback(null);
+    });
 }
 
 function clear(callback) {
@@ -85,43 +107,80 @@ function clear(callback) {
     child_process.exec(cmd, callback);
 }
 
-function query() {
-    const args = Array.prototype.slice.call(arguments);
-    const callback = args[args.length - 1];
+function beginTransaction(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    if (constants.TEST && !gConnectionPool) return callback(new BoxError(BoxError.DATABASE_ERROR, 'database.js not initialized'));
+    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));
 
-    gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
+    gConnectionPool.getConnection(function (error, connection) {
+        if (error) {
+            console.error('Unable to get connection to database. Try again in a bit.', error.message);
+            return setTimeout(beginTransaction.bind(null, callback), 1000);
+        }
+
+        connection.beginTransaction(function (error) {
+            if (error) return callback(error);
+
+            return callback(null, connection);
+        });
+    });
+}
+
+function rollback(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.rollback(function (error) {
+        if (error) console.error(error); // can this happen?
+
+        connection.release();
+        callback(null);
+    });
+}
+
+// FIXME: if commit fails, is it supposed to return an error ?
+function commit(connection, callback) {
+    assert.strictEqual(typeof callback, 'function');
+
+    connection.commit(function (error) {
+        if (error) return rollback(connection, callback);
+
+        connection.release();
+        return callback(null);
+    });
+}
+
+function query() {
+    var args = Array.prototype.slice.call(arguments);
+    var callback = args[args.length - 1];
+    assert.strictEqual(typeof callback, 'function');
+
+    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));
+
+    args[args.length -1 ] = function (error, result) {
+        if (error && error.fatal) {
+            gDefaultConnection = null;
+            setTimeout(reconnect, 1000);
+        }
+
+        callback(error, result);
+    };
+
+    gDefaultConnection.query.apply(gDefaultConnection, args);
 }
 
 function transaction(queries, callback) {
     assert(util.isArray(queries));
     assert.strictEqual(typeof callback, 'function');
 
-    callback = once(callback);
-
-    gConnectionPool.getConnection(function (error, connection) {
+    beginTransaction(function (error, conn) {
         if (error) return callback(error);
 
-        const releaseConnection = (error) => { connection.release(); callback(error); };
+        async.mapSeries(queries, function iterator(query, done) {
+            conn.query(query.query, query.args, done);
+        }, function seriesDone(error, results) {
+            if (error) return rollback(conn, callback.bind(null, error));
 
-        connection.beginTransaction(function (error) {
-            if (error) return releaseConnection(error);
-
-            async.mapSeries(queries, function iterator(query, done) {
-                connection.query(query.query, query.args, done);
-            }, function seriesDone(error, results) {
-                if (error) return connection.rollback(() => releaseConnection(error));
-
-                connection.commit(function (error) {
-                    if (error) return connection.rollback(() => releaseConnection(error));
-
-                    connection.release();
-
-                    callback(null, results);
-                });
-            });
+            commit(conn, callback.bind(null, null, results));
         });
     });
 }
@@ -144,7 +203,7 @@ function exportToFile(file, callback) {
 
     // latest mysqldump enables column stats by default which is not present in MySQL 5.7 server
     // this option must not be set in production cloudrons which still use the old mysqldump
-    const disableColStats = (constants.TEST && require('fs').readFileSync('/etc/lsb-release', 'utf-8').includes('20.04')) ? '--column-statistics=0' : '';
+    const disableColStats = (constants.TEST && process.env.DESKTOP_SESSION !== 'ubuntu') ? '--column-statistics=0' : '';
 
     var cmd = `/usr/bin/mysqldump -h "${gDatabase.hostname}" -u root -p${gDatabase.password} ${disableColStats} --single-transaction --routines --triggers ${gDatabase.name} > "${file}"`;
 

src/dns/caas.js

@@ -0,0 +1,176 @@
+'use strict';
+
+exports = module.exports = {
+    removePrivateFields: removePrivateFields,
+    injectPrivateFields: injectPrivateFields,
+    upsert: upsert,
+    get: get,
+    del: del,
+    wait: wait,
+    verifyDnsConfig: verifyDnsConfig
+};
+
+var assert = require('assert'),
+    BoxError = require('../boxerror.js'),
+    debug = require('debug')('box:dns/caas'),
+    domains = require('../domains.js'),
+    settings = require('../settings.js'),
+    superagent = require('superagent'),
+    util = require('util'),
+    waitForDns = require('./waitfordns.js');
+
+function formatError(response) {
+    return util.format('Caas DNS error [%s] %j', response.statusCode, response.body);
+}
+
+function getFqdn(location, domain) {
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof domain, 'string');
+
+    return (location === '') ? domain : location + '-' + domain;
+}
+
+function removePrivateFields(domainObject) {
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
+
+    // do not return the 'key'. in caas, this is private
+    delete domainObject.fallbackCertificate.key;
+
+    return domainObject;
+}
+
+function injectPrivateFields(newConfig, currentConfig) {
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+}
+
+function upsert(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+
+    let fqdn = location !== '' && type === 'TXT' ? location + '.' + domainObject.domain : getFqdn(location, domainObject.domain);
+
+    debug('add: %s for zone %s of type %s with values %j', location, domainObject.domain, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .post(settings.apiServerOrigin() + '/api/v1/caas/domains/' + fqdn)
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .timeout(30 * 1000)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
+            if (result.statusCode === 420) return callback(new BoxError(BoxError.BUSY));
+            if (result.statusCode !== 201) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            return callback(null);
+        });
+}
+
+function get(domainObject, location, type, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+    const fqdn = location !== '' && type === 'TXT' ? location + '.' + domainObject.domain : getFqdn(location, domainObject.domain);
+
+    debug('get: zoneName: %s subdomain: %s type: %s fqdn: %s', domainObject.domain, location, type, fqdn);
+
+    superagent
+        .get(settings.apiServerOrigin() + '/api/v1/caas/domains/' + fqdn)
+        .query({ token: dnsConfig.token, type: type })
+        .timeout(30 * 1000)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode !== 200) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            return callback(null, result.body.values);
+        });
+}
+
+function del(domainObject, location, type, values, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert(util.isArray(values));
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+    debug('del: %s for zone %s of type %s with values %j', location, domainObject.domain, type, values);
+
+    var data = {
+        type: type,
+        values: values
+    };
+
+    superagent
+        .del(settings.apiServerOrigin() + '/api/v1/caas/domains/' + getFqdn(location, domainObject.domain))
+        .query({ token: dnsConfig.token })
+        .send(data)
+        .timeout(30 * 1000)
+        .end(function (error, result) {
+            if (error && !error.response) return callback(new BoxError(BoxError.NETWORK_ERROR, error.message));
+            if (result.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
+            if (result.statusCode === 420) return callback(new BoxError(BoxError.BUSY));
+            if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+            if (result.statusCode !== 204) return callback(new BoxError(BoxError.EXTERNAL_ERROR, formatError(result)));
+
+            return callback(null);
+        });
+}
+
+function wait(domainObject, location, type, value, options, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof location, 'string');
+    assert.strictEqual(typeof type, 'string');
+    assert.strictEqual(typeof value, 'string');
+    assert(options && typeof options === 'object'); // { interval: 5000, times: 50000 }
+    assert.strictEqual(typeof callback, 'function');
+
+    const fqdn = domains.fqdn(location, domainObject);
+
+    waitForDns(fqdn, domainObject.zoneName, type, value, options, callback);
+}
+
+function verifyDnsConfig(domainObject, callback) {
+    assert.strictEqual(typeof domainObject, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    const dnsConfig = domainObject.config;
+
+    if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
+
+    const ip = '127.0.0.1';
+
+    var credentials = {
+        token: dnsConfig.token,
+        hyphenatedSubdomains: true  // this will ensure we always use them, regardless of passed-in configs
+    };
+
+    const location = 'cloudrontestdns';
+
+    upsert(domainObject, location, 'A', [ ip ], function (error) {
+        if (error) return callback(error);
+
+        debug('verifyDnsConfig: Test A record added');
+
+        del(domainObject, location, 'A', [ ip ], function (error) {
+            if (error) return callback(error);
+
+            debug('verifyDnsConfig: Test A record removed again');
+
+            callback(null, credentials);
+        });
+    });
+}

src/dns/cloudflare.js

@@ -13,7 +13,6 @@ exports = module.exports = {
 var assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/cloudflare'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -26,12 +25,12 @@ var assert = require('assert'),
 var CLOUDFLARE_ENDPOINT = 'https://api.cloudflare.com/client/v4';
 
 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
 function translateRequestError(result, callback) {
@@ -40,14 +39,9 @@ function translateRequestError(result, callback) {
 
     if (result.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, util.format('%s %j', result.statusCode, 'API does not exist')));
     if (result.statusCode === 422) return callback(new BoxError(BoxError.BAD_FIELD, result.body.message));
-    if (result.statusCode === 400 || result.statusCode === 401 || result.statusCode === 403) {
-        let message = 'Unknown error';
-        if (typeof result.body.error === 'string') {
-            message = `message: ${result.body.error} statusCode: ${result.statusCode}`;
-        } else if (Array.isArray(result.body.errors) && result.body.errors.length > 0) {
-            let error = result.body.errors[0];
-            message = `message: ${error.message} statusCode: ${result.statusCode} code:${error.code}`;
-        }
+    if ((result.statusCode === 400 || result.statusCode === 401 || result.statusCode === 403) && result.body.errors.length > 0) {
+        let error = result.body.errors[0];
+        let message = `message: ${error.message} statusCode: ${result.statusCode} code:${error.code}`;
         return callback(new BoxError(BoxError.ACCESS_DENIED, message));
     }
 
@@ -290,7 +284,7 @@ function verifyDnsConfig(domainObject, callback) {
     if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
 
     if (dnsConfig.tokenType === 'GlobalApiKey') {
-        if (typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
     }
 
     const ip = '127.0.0.1';

src/dns/digitalocean.js

@@ -13,7 +13,6 @@ exports = module.exports = {
 var assert = require('assert'),
     async = require('async'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/digitalocean'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -29,12 +28,12 @@ function formatError(response) {
 }
 
 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
 function getInternal(dnsConfig, zoneName, name, type, callback) {

src/dns/gandi.js

@@ -12,7 +12,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/gandi'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -27,12 +26,12 @@ function formatError(response) {
 }
 
 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
 function upsert(domainObject, location, type, values, callback) {

src/dns/gcdns.js

@@ -12,7 +12,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/gcdns'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -22,12 +21,12 @@ var assert = require('assert'),
     _ = require('underscore');
 
 function removePrivateFields(domainObject) {
-    domainObject.config.credentials.private_key = constants.SECRET_PLACEHOLDER;
+    domainObject.config.credentials.private_key = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.credentials.private_key === constants.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
+    if (newConfig.credentials.private_key === domains.SECRET_PLACEHOLDER && currentConfig.credentials) newConfig.credentials.private_key = currentConfig.credentials.private_key;
 }
 
 function getDnsCredentials(dnsConfig) {

src/dns/godaddy.js

@@ -12,7 +12,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/godaddy'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -33,12 +32,12 @@ function formatError(response) {
 }
 
 function removePrivateFields(domainObject) {
-    domainObject.config.apiSecret = constants.SECRET_PLACEHOLDER;
+    domainObject.config.apiSecret = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.apiSecret === constants.SECRET_PLACEHOLDER) newConfig.apiSecret = currentConfig.apiSecret;
+    if (newConfig.apiSecret === domains.SECRET_PLACEHOLDER) newConfig.apiSecret = currentConfig.apiSecret;
 }
 
 function upsert(domainObject, location, type, values, callback) {

src/dns/interface.js

@@ -21,13 +21,13 @@ var assert = require('assert'),
     util = require('util');
 
 function removePrivateFields(domainObject) {
-    // in-place removal of tokens and api keys with constants.SECRET_PLACEHOLDER
+    // in-place removal of tokens and api keys with domains.SECRET_PLACEHOLDER
     return domainObject;
 }
 
 // eslint-disable-next-line no-unused-vars
 function injectPrivateFields(newConfig, currentConfig) {
-    // in-place injection of tokens and api keys which came in with constants.SECRET_PLACEHOLDER
+    // in-place injection of tokens and api keys which came in with domains.SECRET_PLACEHOLDER
 }
 
 function upsert(domainObject, location, type, values, callback) {

src/dns/linode.js

@@ -12,7 +12,6 @@ exports = module.exports = {
 
 let async = require('async'),
     assert = require('assert'),
-    constants = require('../constants.js'),
     BoxError = require('../boxerror.js'),
     debug = require('debug')('box:dns/linode'),
     dns = require('../native-dns.js'),
@@ -28,12 +27,12 @@ function formatError(response) {
 }
 
 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
 function getZoneId(dnsConfig, zoneName, callback) {

src/dns/namecheap.js

@@ -12,7 +12,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecheap'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -26,12 +25,12 @@ var assert = require('assert'),
 const ENDPOINT = 'https://api.namecheap.com/xml.response';
 
 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
 function getQuery(dnsConfig, callback) {

src/dns/namecom.js

@@ -12,7 +12,6 @@ exports = module.exports = {
 
 var assert = require('assert'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/namecom'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -28,12 +27,12 @@ function formatError(response) {
 }
 
 function removePrivateFields(domainObject) {
-    domainObject.config.token = constants.SECRET_PLACEHOLDER;
+    domainObject.config.token = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.token === constants.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
+    if (newConfig.token === domains.SECRET_PLACEHOLDER) newConfig.token = currentConfig.token;
 }
 
 function addRecord(dnsConfig, zoneName, name, type, values, callback) {
@@ -55,10 +54,6 @@ function addRecord(dnsConfig, zoneName, name, type, values, callback) {
     if (type === 'MX') {
         data.priority = parseInt(values[0].split(' ')[0], 10);
         data.answer = values[0].split(' ')[1];
-    } else if (type === 'TXT') {
-        // we have to strip the quoting for some odd reason for name.com! If you change that also change updateRecord
-        let tmp = values[0];
-        data.answer = tmp.indexOf('"') === 0 && tmp.lastIndexOf('"') === tmp.length-1 ? tmp.slice(1, tmp.length-1) : tmp;
     } else {
         data.answer = values[0];
     }
@@ -96,10 +91,6 @@ function updateRecord(dnsConfig, zoneName, recordId, name, type, values, callbac
     if (type === 'MX') {
         data.priority = parseInt(values[0].split(' ')[0], 10);
         data.answer = values[0].split(' ')[1];
-    } else if (type === 'TXT') {
-        // we have to strip the quoting for some odd reason for name.com! If you change that also change addRecord
-        let tmp = values[0];
-        data.answer = tmp.indexOf('"') === 0 && tmp.lastIndexOf('"') === tmp.length-1 ? tmp.slice(1, tmp.length-1) : tmp;
     } else {
         data.answer = values[0];
     }

src/dns/route53.js

@@ -13,7 +13,6 @@ exports = module.exports = {
 var assert = require('assert'),
     AWS = require('aws-sdk'),
     BoxError = require('../boxerror.js'),
-    constants = require('../constants.js'),
     debug = require('debug')('box:dns/route53'),
     dns = require('../native-dns.js'),
     domains = require('../domains.js'),
@@ -22,12 +21,12 @@ var assert = require('assert'),
     _ = require('underscore');
 
 function removePrivateFields(domainObject) {
-    domainObject.config.secretAccessKey = constants.SECRET_PLACEHOLDER;
+    domainObject.config.secretAccessKey = domains.SECRET_PLACEHOLDER;
     return domainObject;
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.secretAccessKey === constants.SECRET_PLACEHOLDER) newConfig.secretAccessKey = currentConfig.secretAccessKey;
+    if (newConfig.secretAccessKey === domains.SECRET_PLACEHOLDER) newConfig.secretAccessKey = currentConfig.secretAccessKey;
 }
 
 function getDnsCredentials(dnsConfig) {
@@ -281,14 +280,13 @@ function verifyDnsConfig(domainObject, callback) {
             }
 
             const location = 'cloudrontestdns';
-            const newDomainObject = Object.assign({ }, domainObject, { config: credentials });
 
-            upsert(newDomainObject, location, 'A', [ ip ], function (error) {
+            upsert(domainObject, location, 'A', [ ip ], function (error) {
                 if (error) return callback(error);
 
                 debug('verifyDnsConfig: Test A record added');
 
-                del(newDomainObject, location, 'A', [ ip ], function (error) {
+                del(domainObject, location, 'A', [ ip ], function (error) {
                     if (error) return callback(error);
 
                     debug('verifyDnsConfig: Test A record removed again');

src/docker.js

@@ -6,6 +6,8 @@ exports = module.exports = {
     injectPrivateFields: injectPrivateFields,
     removePrivateFields: removePrivateFields,
 
+    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8),
+
     ping: ping,
 
     info: info,
@@ -53,6 +55,12 @@ const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
 
+function debugApp(app) {
+    assert(typeof app === 'object');
+
+    debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
+}
+
 function testRegistryConfig(auth, callback) {
     assert.strictEqual(typeof auth, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -65,13 +73,13 @@ function testRegistryConfig(auth, callback) {
 }
 
 function injectPrivateFields(newConfig, currentConfig) {
-    if (newConfig.password === constants.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password;
+    if (newConfig.password === exports.SECRET_PLACEHOLDER) newConfig.password = currentConfig.password;
 }
 
 function removePrivateFields(registryConfig) {
     assert.strictEqual(typeof registryConfig, 'object');
 
-    if (registryConfig.password) registryConfig.password = constants.SECRET_PLACEHOLDER;
+    if (registryConfig.password) registryConfig.password = exports.SECRET_PLACEHOLDER;
 
     return registryConfig;
 }
@@ -180,19 +188,6 @@ function downloadImage(manifest, callback) {
     }, callback);
 }
 
-function getBindsSync(app) {
-    assert.strictEqual(typeof app, 'object');
-
-    let binds = [];
-
-    for (let name of Object.keys(app.binds)) {
-        const bind = app.binds[name];
-        binds.push(`${bind.hostPath}:/media/${name}:${bind.readOnly ? 'ro' : 'rw'}`);
-    }
-
-    return binds;
-}
-
 function createSubcontainer(app, name, cmd, options, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
@@ -282,7 +277,6 @@ function createSubcontainer(app, name, cmd, options, callback) {
             },
             HostConfig: {
                 Mounts: addons.getMountsSync(app, app.manifest.addons),
-                Binds: getBindsSync(app), // ideally, we have to use 'Mounts' but we have to create volumes then
                 LogConfig: {
                     Type: 'syslog',
                     Config: {
@@ -305,9 +299,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 NetworkMode: 'cloudron', // user defined bridge network
                 Dns: ['172.18.0.1'], // use internal dns
                 DnsSearch: ['.'], // use internal dns
-                SecurityOpt: [ 'apparmor=docker-cloudron-app' ],
-                CapAdd: [],
-                CapDrop: []
+                SecurityOpt: [ 'apparmor=docker-cloudron-app' ]
             },
             NetworkingConfig: {
                 EndpointsConfig: {
@@ -319,14 +311,16 @@ function createSubcontainer(app, name, cmd, options, callback) {
         };
 
         var capabilities = manifest.capabilities || [];
-
-        // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
-        if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW');
-        if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
-        if (!capabilities.includes('ping')) containerOptions.HostConfig.CapDrop.push('NET_RAW'); // NET_RAW is included by default by Docker
+        if (capabilities.includes('net_admin')) {
+            containerOptions.HostConfig.CapAdd = [
+                'NET_ADMIN'
+            ];
+        }
 
         containerOptions = _.extend(containerOptions, options);
 
+        debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
+
         gConnection.createContainer(containerOptions, function (error, container) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
@@ -344,6 +338,7 @@ function startContainer(containerId, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var container = gConnection.getContainer(containerId);
+    debug('Starting container %s', containerId);
 
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -359,6 +354,7 @@ function restartContainer(containerId, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);
 
     container.restart(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -379,6 +375,7 @@ function stopContainer(containerId, callback) {
     }
 
     var container = gConnection.getContainer(containerId);
+    debug('Stopping container %s', containerId);
 
     var options = {
         t: 10 // wait for 10 seconds before killing it
@@ -387,9 +384,13 @@ function stopContainer(containerId, callback) {
     container.stop(options, function (error) {
         if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message));
 
-        container.wait(function (error/*, data */) {
+        debug('Waiting for container ' + containerId);
+
+        container.wait(function (error, data) {
             if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message));
 
+            debug('Container %s stopped with status code [%s]', containerId, data ? String(data.StatusCode) : '');
+
             return callback(null);
         });
     });
@@ -399,6 +400,8 @@ function deleteContainer(containerId, callback) {
     assert(!containerId || typeof containerId === 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    debug('deleting container %s', containerId);
+
     if (containerId === null) return callback(null);
 
     var container = gConnection.getContainer(containerId);
@@ -425,6 +428,8 @@ function deleteContainers(appId, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    debug('deleting containers of %s', appId);
+
     let labels = [ 'appId=' + appId ];
     if (options.managedOnly) labels.push('isCloudronManaged=true');
 
@@ -441,6 +446,8 @@ function stopContainers(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
+    debug('Stopping containers of %s', appId);
+
     gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
@@ -507,7 +514,7 @@ function inspect(containerId, callback) {
     var container = gConnection.getContainer(containerId);
 
     container.inspect(function (error, result) {
-        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND, `Unable to find container ${containerId}`));
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         callback(null, result);
@@ -566,10 +573,10 @@ function memoryUsage(containerId, callback) {
     });
 }
 
-function createVolume(name, volumeDataDir, labels, callback) {
+function createVolume(app, name, volumeDataDir, callback) {
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof volumeDataDir, 'string');
-    assert.strictEqual(typeof labels, 'object');
     assert.strictEqual(typeof callback, 'function');
 
     const volumeOptions = {
@@ -580,7 +587,10 @@ function createVolume(name, volumeDataDir, labels, callback) {
             device: volumeDataDir,
             o: 'bind'
         },
-        Labels: labels
+        Labels: {
+            'fqdn': app.fqdn,
+            'appId': app.id
+        },
     };
 
     // requires sudo because the path can be outside appsdata
@@ -595,7 +605,8 @@ function createVolume(name, volumeDataDir, labels, callback) {
     });
 }
 
-function clearVolume(name, options, callback) {
+function clearVolume(app, name, options, callback) {
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -615,13 +626,14 @@ function clearVolume(name, options, callback) {
 }
 
 // this only removes the volume and not the data
-function removeVolume(name, callback) {
+function removeVolume(app, name, callback) {
+    assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     let volume = gConnection.getVolume(name);
     volume.remove(function (error) {
-        if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume: ${error.message}`));
+        if (error && error.statusCode !== 404) return callback(new BoxError(BoxError.DOCKER_ERROR, `removeVolume: Error removing volume of ${app.id} ${error.message}`));
 
         callback();
     });

src/dockerproxy.js

@@ -23,6 +23,11 @@ var apps = require('./apps.js'),
 var gHttpServer = null;
 
 function authorizeApp(req, res, next) {
+    // TODO add here some authorization
+    // - block apps not using the docker addon
+    // - block calls regarding platform containers
+    // - only allow managing and inspection of containers belonging to the app
+
     // make the tests pass for now
     if (constants.TEST) {
         req.app = { id: 'testappid' };
@@ -55,12 +60,10 @@ function attachDockerRequest(req, res, next) {
         // Force node to send out the headers, this is required for the /container/wait api to make the docker cli proceed
         res.write(' ');
 
-        dockerResponse.on('error', function (error) { debug('dockerResponse error:', error); });
+        dockerResponse.on('error', function (error) { console.error('dockerResponse error:', error); });
         dockerResponse.pipe(res, { end: true });
     });
 
-    req.dockerRequest.on('error', () => {}); // abort() throws
-
     next();
 }
 
@@ -71,21 +74,22 @@ function containersCreate(req, res, next) {
     safe.set(req.body, 'Labels',  _.extend({ }, safe.query(req.body, 'Labels'), { appId: req.app.id, isCloudronManaged: String(false) }));    // overwrite the app id to track containers of an app
     safe.set(req.body, 'HostConfig.LogConfig', { Type: 'syslog', Config: { 'tag': req.app.id, 'syslog-address': 'udp://127.0.0.1:2514', 'syslog-format': 'rfc5424' }});
 
-    const appDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'data');
+    const appDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'data'),
+        dockerDataDir = path.join(paths.APPS_DATA_DIR, req.app.id, 'docker');
 
-    debug('Original bind mounts:', req.body.HostConfig.Binds);
+    debug('Original volume binds:', req.body.HostConfig.Binds);
 
     let binds = [];
     for (let bind of (req.body.HostConfig.Binds || [])) {
-        if (!bind.startsWith('/app/data/')) {
-            req.dockerRequest.abort();
-            return next(new HttpError(400, 'Binds must be under /app/data/'));
-        }
-
-        binds.push(bind.replace(new RegExp('^/app/data/'), appDataDir + '/'));
+        if (bind.startsWith(appDataDir)) binds.push(bind); // eclipse will inspect docker to find out the host folders and pass that to child containers
+        else if (bind.startsWith('/app/data')) binds.push(bind.replace(new RegExp('^/app/data'), appDataDir));
+        else binds.push(`${dockerDataDir}/${bind}`);
     }
 
-    debug('Rewritten bind mounts:', binds);
+    // cleanup the paths from potential double slashes
+    binds = binds.map(function (bind) { return bind.replace(/\/+/g, '/'); });
+
+    debug('Rewritten volume binds:', binds);
     safe.set(req.body, 'HostConfig.Binds', binds);
 
     let plainBody = JSON.stringify(req.body);
@@ -113,9 +117,6 @@ function start(callback) {
     assert(gHttpServer === null, 'Already started');
 
     let json = middleware.json({ strict: true });
-
-    // we protect container create as the app/admin can otherwise mount random paths (like the ghost file)
-    // protected other paths is done by preventing install/exec access of apps using docker addon
     let router = new express.Router();
     router.post('/:version/containers/create', containersCreate);
 
@@ -136,7 +137,7 @@ function start(callback) {
         .use(middleware.lastMile());
 
     gHttpServer = http.createServer(proxyServer);
-    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '172.18.0.1', callback);
+    gHttpServer.listen(constants.DOCKER_PROXY_PORT, '0.0.0.0', callback);
 
     // Overwrite the default 2min request timeout. This is required for large builds for example
     gHttpServer.setTimeout(60 * 60 * 1000);

src/domaindb.js

@@ -51,22 +51,17 @@ function getAll(callback) {
     });
 }
 
-function add(name, data, callback) {
+function add(name, domain, callback) {
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof data, 'object');
-    assert.strictEqual(typeof data.zoneName, 'string');
-    assert.strictEqual(typeof data.provider, 'string');
-    assert.strictEqual(typeof data.config, 'object');
-    assert.strictEqual(typeof data.tlsConfig, 'object');
+    assert.strictEqual(typeof domain, 'object');
+    assert.strictEqual(typeof domain.zoneName, 'string');
+    assert.strictEqual(typeof domain.provider, 'string');
+    assert.strictEqual(typeof domain.config, 'object');
+    assert.strictEqual(typeof domain.tlsConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let queries = [
-        { query: 'INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', args: [ name, data.zoneName, data.provider, JSON.stringify(data.config), JSON.stringify(data.tlsConfig) ] },
-        { query: 'INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', args: [ name, data.dkimSelector || 'cloudron' ] },
-    ];
-
-    database.transaction(queries, function (error) {
-        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'Domain already exists'));
+    database.query('INSERT INTO domains (domain, zoneName, provider, configJson, tlsConfigJson) VALUES (?, ?, ?, ?, ?)', [ name, domain.zoneName, domain.provider, JSON.stringify(domain.config), JSON.stringify(domain.tlsConfig) ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         callback(null);
@@ -105,12 +100,7 @@ function del(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    let queries = [
-        { query: 'DELETE FROM mail WHERE domain = ?', args: [ domain ] },
-        { query: 'DELETE FROM domains WHERE domain = ?', args: [ domain ] },
-    ];
-
-    database.transaction(queries, function (error, results) {
+    database.query('DELETE FROM domains WHERE domain=?', [ domain ], function (error, result) {
         if (error && error.code === 'ER_ROW_IS_REFERENCED_2') {
             if (error.message.indexOf('apps_mailDomain_constraint') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by an app or the mailbox of an app. Check the domains of apps and the Email section of each app.'));
             if (error.message.indexOf('subdomains') !== -1) return callback(new BoxError(BoxError.CONFLICT, 'Domain is in use by one or more app(s).'));
@@ -118,9 +108,8 @@ function del(domain, callback) {
 
             return callback(new BoxError(BoxError.CONFLICT, error.message));
         }
-
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (results[1].affectedRows !== 1) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
+        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Domain not found'));
 
         callback(null);
     });

src/domains.js

@@ -28,7 +28,9 @@ module.exports = exports = {
 
     checkDnsRecords: checkDnsRecords,
 
-    prepareDashboardDomain: prepareDashboardDomain
+    prepareDashboardDomain: prepareDashboardDomain,
+
+    SECRET_PLACEHOLDER: String.fromCharCode(0x25CF).repeat(8)
 };
 
 var assert = require('assert'),
@@ -38,7 +40,6 @@ var assert = require('assert'),
     debug = require('debug')('box:domains'),
     domaindb = require('./domaindb.js'),
     eventlog = require('./eventlog.js'),
-    mail = require('./mail.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
     settings = require('./settings.js'),
@@ -47,13 +48,12 @@ var assert = require('assert'),
     util = require('util'),
     _ = require('underscore');
 
-const NOOP_CALLBACK = function (error) { if (error) debug(error); };
-
 // choose which subdomain backend we use for test purpose we use route53
 function api(provider) {
     assert.strictEqual(typeof provider, 'string');
 
     switch (provider) {
+    case 'caas': return require('./dns/caas.js');
     case 'cloudflare': return require('./dns/cloudflare.js');
     case 'route53': return require('./dns/route53.js');
     case 'gcdns': return require('./dns/gcdns.js');
@@ -148,9 +148,10 @@ function validateTlsConfig(tlsConfig, dnsProvider) {
     case 'letsencrypt-prod':
     case 'letsencrypt-staging':
     case 'fallback':
+    case 'caas':
         break;
     default:
-        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be fallback, letsencrypt-prod/staging', { field: 'tlsProvider' });
+        return new BoxError(BoxError.BAD_FIELD, 'tlsConfig.provider must be caas, fallback, letsencrypt-prod/staging', { field: 'tlsProvider' });
     }
 
     if (tlsConfig.wildcard) {
@@ -170,7 +171,7 @@ function add(domain, data, auditSource, callback) {
     assert.strictEqual(typeof data.tlsConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let { zoneName, provider, config, fallbackCertificate, tlsConfig, dkimSelector } = data;
+    let { zoneName, provider, config, fallbackCertificate, tlsConfig } = data;
 
     if (!tld.isValid(domain)) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
     if (domain.endsWith('.')) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid domain', { field: 'domain' }));
@@ -193,12 +194,10 @@ function add(domain, data, auditSource, callback) {
     let error = validateTlsConfig(tlsConfig, provider);
     if (error) return callback(error);
 
-    if (!dkimSelector) dkimSelector = 'cloudron-' + settings.adminDomain().replace(/\./g, '');
-
     verifyDnsConfig(config, domain, zoneName, provider, function (error, sanitizedConfig) {
         if (error) return callback(error);
 
-        domaindb.add(domain, { zoneName, provider, config: sanitizedConfig, tlsConfig, dkimSelector }, function (error) {
+        domaindb.add(domain, { zoneName: zoneName, provider: provider, config: sanitizedConfig, tlsConfig: tlsConfig }, function (error) {
             if (error) return callback(error);
 
             reverseProxy.setFallbackCertificate(domain, fallbackCertificate, function (error) {
@@ -206,8 +205,6 @@ function add(domain, data, auditSource, callback) {
 
                 eventlog.add(eventlog.ACTION_DOMAIN_ADD, auditSource, { domain, zoneName, provider });
 
-                mail.onDomainAdded(domain, NOOP_CALLBACK);
-
                 callback();
             });
         });
@@ -317,8 +314,6 @@ function del(domain, auditSource, callback) {
 
         eventlog.add(eventlog.ACTION_DOMAIN_REMOVE, auditSource, { domain });
 
-        mail.onDomainRemoved(domain, NOOP_CALLBACK);
-
         return callback(null);
     });
 }

src/externalldap.js

@@ -20,9 +20,7 @@ var assert = require('assert'),
     BoxError = require('./boxerror.js'),
     constants = require('./constants.js'),
     debug = require('debug')('box:externalldap'),
-    groups = require('./groups.js'),
     ldap = require('ldapjs'),
-    once = require('once'),
     settings = require('./settings.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js');
@@ -42,14 +40,14 @@ function translateUser(ldapConfig, ldapUser) {
 
     return {
         username: ldapUser[ldapConfig.usernameField],
-        email: ldapUser.mail || ldapUser.mailPrimaryAddress,
+        email: ldapUser.mail,
         displayName: ldapUser.cn // user.giveName + ' ' + user.sn
     };
 }
 
 function validUserRequirements(user) {
     if (!user.username || !user.email || !user.displayName) {
-        debug(`[Invalid LDAP user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+        debug(`[LDAP user empty username/email/displayName] username=${user.username} email=${user.email} displayName=${user.displayName}`);
         return false;
     } else {
         return true;
@@ -57,95 +55,40 @@ function validUserRequirements(user) {
 }
 
 // performs service bind if required
-function getClient(externalLdapConfig, doBindAuth, callback) {
+function getClient(externalLdapConfig, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof doBindAuth, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
-    // ensure we only callback once since we also have to listen to client.error events
-    callback = once(callback);
-
     // basic validation to not crash
     try { ldap.parseDN(externalLdapConfig.baseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid baseDn')); }
     try { ldap.parseFilter(externalLdapConfig.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
 
-    var config = {
-        url: externalLdapConfig.url,
-        tlsOptions: {
-            rejectUnauthorized: externalLdapConfig.acceptSelfSignedCerts ? false : true
-        }
-    };
-
     var client;
     try {
-        client = ldap.createClient(config);
+        client = ldap.createClient({ url: externalLdapConfig.url });
     } catch (e) {
         if (e instanceof ldap.ProtocolError) return callback(new BoxError(BoxError.BAD_FIELD, 'url protocol is invalid'));
         return callback(new BoxError(BoxError.INTERNAL_ERROR, e));
     }
 
-    // ensure we don't just crash
-    client.on('error', function (error) {
-        callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-    });
-
-    // skip bind auth if none exist or if not wanted
-    if (!externalLdapConfig.bindDn || !doBindAuth) return callback(null, client);
+    if (!externalLdapConfig.bindDn) return callback(null, client);
 
     client.bind(externalLdapConfig.bindDn, externalLdapConfig.bindPassword, function (error) {
         if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
         if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-        callback(null, client);
+        callback(null, client, externalLdapConfig);
     });
 }
 
-function ldapGetByDN(externalLdapConfig, dn, callback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof dn, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getClient(externalLdapConfig, true, function (error, client) {
-        if (error) return callback(error);
-
-        let searchOptions = {
-            paged: true,
-            scope: 'sub'    // We may have to make this configurable
-        };
-
-        debug(`Get object at ${dn}`);
-
-        // basic validation to not crash
-        try { ldap.parseDN(dn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid DN')); }
-
-        client.search(dn, searchOptions, function (error, result) {
-            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            let ldapObjects = [];
-
-            result.on('searchEntry', entry => ldapObjects.push(entry.object));
-            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
-
-            result.on('end', function (result) {
-                client.unbind();
-
-                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
-                if (ldapObjects.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
-
-                callback(null, ldapObjects[0]);
-            });
-        });
-    });
-}
 
 // TODO support search by email
-function ldapUserSearch(externalLdapConfig, options, callback) {
+function ldapSearch(externalLdapConfig, options, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    getClient(externalLdapConfig, true, function (error, client) {
+    getClient(externalLdapConfig, function (error, client) {
         if (error) return callback(error);
 
         let searchOptions = {
@@ -181,48 +124,6 @@ function ldapUserSearch(externalLdapConfig, options, callback) {
     });
 }
 
-function ldapGroupSearch(externalLdapConfig, options, callback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
-
-    getClient(externalLdapConfig, true, function (error, client) {
-        if (error) return callback(error);
-
-        let searchOptions = {
-            paged: true,
-            scope: 'sub'    // We may have to make this configurable
-        };
-
-        if (externalLdapConfig.groupFilter) searchOptions.filter = ldap.parseFilter(externalLdapConfig.groupFilter);
-
-        if (options.filter) { // https://github.com/ldapjs/node-ldapjs/blob/master/docs/filters.md
-            let extraFilter = ldap.parseFilter(options.filter);
-            searchOptions.filter = new ldap.AndFilter({ filters: [ extraFilter, searchOptions.filter ] });
-        }
-
-        debug(`Listing groups at ${externalLdapConfig.groupBaseDn} with filter ${searchOptions.filter.toString()}`);
-
-        client.search(externalLdapConfig.groupBaseDn, searchOptions, function (error, result) {
-            if (error instanceof ldap.NoSuchObjectError) return callback(new BoxError(BoxError.NOT_FOUND));
-            if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-            let ldapGroups = [];
-
-            result.on('searchEntry', entry => ldapGroups.push(entry.object));
-            result.on('error', error => callback(new BoxError(BoxError.EXTERNAL_ERROR, error)));
-
-            result.on('end', function (result) {
-                client.unbind();
-
-                if (result.status !== 0) return callback(new BoxError(BoxError.EXTERNAL_ERROR, 'Server returned status ' + result.status));
-
-                callback(null, ldapGroups);
-            });
-        });
-    });
-}
-
 function testConfig(config, callback) {
     assert.strictEqual(typeof config, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -240,20 +141,7 @@ function testConfig(config, callback) {
     if (!config.filter) return callback(new BoxError(BoxError.BAD_FIELD, 'filter must not be empty'));
     try { ldap.parseFilter(config.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
 
-    if ('syncGroups' in config && typeof config.syncGroups !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'syncGroups must be a boolean'));
-    if ('acceptSelfSignedCerts' in config && typeof config.acceptSelfSignedCerts !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean'));
-
-    if (config.syncGroups) {
-        if (!config.groupBaseDn) return callback(new BoxError(BoxError.BAD_FIELD, 'groupBaseDn must not be empty'));
-        try { ldap.parseDN(config.groupBaseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid groupBaseDn')); }
-
-        if (!config.groupFilter) return callback(new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty'));
-        try { ldap.parseFilter(config.groupFilter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid groupFilter')); }
-
-        if (!config.groupnameField || typeof config.groupnameField !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'groupFilter must not be empty'));
-    }
-
-    getClient(config, true, function (error, client) {
+    getClient(config, function (error, client) {
         if (error) return callback(error);
 
         var opts = {
@@ -279,7 +167,7 @@ function search(identifier, callback) {
         if (error) return callback(error);
         if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
 
-        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
             if (error) return callback(error);
 
             // translate ldap properties to ours
@@ -300,7 +188,7 @@ function createAndVerifyUserIfNotExist(identifier, password, callback) {
         if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
         if (!externalLdapConfig.autoCreate) return callback(new BoxError(BoxError.BAD_STATE, 'auto create not enabled'));
 
-        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${identifier}` }, function (error, ldapUsers) {
             if (error) return callback(error);
             if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
             if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
@@ -310,7 +198,7 @@ function createAndVerifyUserIfNotExist(identifier, password, callback) {
 
             users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_AUTO_CREATE, function (error, user) {
                 if (error) {
-                    debug(`createAndVerifyUserIfNotExist: Failed to auto create user ${user.username}`, error);
+                    console.error('Failed to auto create user', user.username, error);
                     return callback(new BoxError(BoxError.INTERNAL_ERROR));
                 }
 
@@ -332,20 +220,17 @@ function verifyPassword(user, password, callback) {
         if (error) return callback(error);
         if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
 
-        ldapUserSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` }, function (error, ldapUsers) {
+        ldapSearch(externalLdapConfig, { filter: `${externalLdapConfig.usernameField}=${user.username}` }, function (error, ldapUsers) {
             if (error) return callback(error);
             if (ldapUsers.length === 0) return callback(new BoxError(BoxError.NOT_FOUND));
             if (ldapUsers.length > 1) return callback(new BoxError(BoxError.CONFLICT));
 
-            getClient(externalLdapConfig, false, function (error, client) {
-                if (error) return callback(error);
+            let client = ldap.createClient({ url: externalLdapConfig.url });
+            client.bind(ldapUsers[0].dn, password, function (error) {
+                if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
+                if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
 
-                client.bind(ldapUsers[0].dn, password, function (error) {
-                    if (error instanceof ldap.InvalidCredentialsError) return callback(new BoxError(BoxError.INVALID_CREDENTIALS));
-                    if (error) return callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
-
-                    callback(null, translateUser(externalLdapConfig, ldapUsers[0]));
-                });
+                callback(null, translateUser(externalLdapConfig, ldapUsers[0]));
             });
         });
     });
@@ -370,197 +255,6 @@ function startSyncer(callback) {
     });
 }
 
-function syncUsers(externalLdapConfig, progressCallback, callback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    ldapUserSearch(externalLdapConfig, {}, function (error, ldapUsers) {
-        if (error) return callback(error);
-
-        debug(`Found ${ldapUsers.length} users`);
-
-        let percent = 10;
-        let step = 30/(ldapUsers.length+1); // ensure no divide by 0
-
-        // we ignore all errors here and just log them for now
-        async.eachSeries(ldapUsers, function (user, iteratorCallback) {
-            user = translateUser(externalLdapConfig, user);
-
-            if (!validUserRequirements(user)) return iteratorCallback();
-
-            percent += step;
-            progressCallback({ percent, message: `Syncing... ${user.username}` });
-
-            users.getByUsername(user.username, function (error, result) {
-                if (error && error.reason !== BoxError.NOT_FOUND) return iteratorCallback(error);
-
-                if (!result) {
-                    debug(`[adding user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
-
-                    users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
-                        if (error) debug('syncUsers: Failed to create user', user, error.message);
-                        iteratorCallback();
-                    });
-                } else if (result.source !== 'ldap') {
-                    debug(`[conflicting user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
-
-                    iteratorCallback();
-                } else if (result.email !== user.email || result.displayName !== user.displayName) {
-                    debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
-
-                    users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
-                        if (error) debug('Failed to update user', user, error);
-
-                        iteratorCallback();
-                    });
-                } else {
-                    // user known and up-to-date
-                    debug(`[up-to-date user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
-
-                    iteratorCallback();
-                }
-            });
-        }, callback);
-    });
-}
-
-function syncGroups(externalLdapConfig, progressCallback, callback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!externalLdapConfig.syncGroups) {
-        debug('Group sync is disabled');
-        progressCallback({ percent: 70, message: 'Skipping group sync...' });
-        return callback(null, []);
-    }
-
-    ldapGroupSearch(externalLdapConfig, {}, function (error, ldapGroups) {
-        if (error) return callback(error);
-
-        debug(`Found ${ldapGroups.length} groups`);
-
-        let percent = 40;
-        let step = 30/(ldapGroups.length+1); // ensure no divide by 0
-
-        // we ignore all non internal errors here and just log them for now
-        async.eachSeries(ldapGroups, function (ldapGroup, iteratorCallback) {
-            var groupName = ldapGroup[externalLdapConfig.groupnameField];
-            if (!groupName) return iteratorCallback();
-            // some servers return empty array for unknown properties :-/
-            if (typeof groupName !== 'string') return iteratorCallback();
-
-            // groups are lowercase
-            groupName = groupName.toLowerCase();
-
-            percent += step;
-            progressCallback({ percent, message: `Syncing... ${groupName}` });
-
-            groups.getByName(groupName, function (error, result) {
-                if (error && error.reason !== BoxError.NOT_FOUND) return iteratorCallback(error);
-
-                if (!result) {
-                    debug(`[adding group] groupname=${groupName}`);
-
-                    groups.create(groupName, 'ldap', function (error) {
-                        if (error) debug('syncGroups: Failed to create group', groupName, error);
-                        iteratorCallback();
-                    });
-                } else {
-                    debug(`[up-to-date group] groupname=${groupName}`);
-
-                    iteratorCallback();
-                }
-            });
-        }, function (error) {
-            if (error) return callback(error);
-
-            debug('sync: ldap sync is done', error);
-
-            callback(error);
-        });
-    });
-}
-
-function syncGroupUsers(externalLdapConfig, progressCallback, callback) {
-    assert.strictEqual(typeof externalLdapConfig, 'object');
-    assert.strictEqual(typeof progressCallback, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    if (!externalLdapConfig.syncGroups) {
-        debug('Group users sync is disabled');
-        progressCallback({ percent: 99, message: 'Skipping group users sync...' });
-        return callback(null, []);
-    }
-
-    groups.getAll(function (error, result) {
-        if (error) return callback(error);
-
-        var ldapGroups = result.filter(function (g) { return g.source === 'ldap'; });
-        debug(`Found ${ldapGroups.length} groups to sync users`);
-
-        async.eachSeries(ldapGroups, function (group, iteratorCallback) {
-            debug(`Sync users for group ${group.name}`);
-
-            ldapGroupSearch(externalLdapConfig, {}, function (error, result) {
-                if (error) return callback(error);
-                if (!result || result.length === 0) {
-                    debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
-                    return callback();
-                }
-
-                // since our group names are lowercase we cannot use potentially case matching ldap filters
-                let found = result.find(function (r) {
-                    if (!r[externalLdapConfig.groupnameField]) return false;
-                    return r[externalLdapConfig.groupnameField].toLowerCase() === group.name;
-                });
-
-                if (!found) {
-                    debug(`syncGroupUsers: Unable to find group ${group.name} ignoring for now.`);
-                    return callback();
-                }
-
-                var ldapGroupMembers = found.member || found.uniqueMember || [];
-
-                // if only one entry is in the group ldap returns a string, not an array!
-                if (typeof ldapGroupMembers === 'string') ldapGroupMembers = [ ldapGroupMembers ];
-
-                debug(`Group ${group.name} has ${ldapGroupMembers.length} members.`);
-
-                async.eachSeries(ldapGroupMembers, function (memberDn, iteratorCallback) {
-                    ldapGetByDN(externalLdapConfig, memberDn, function (error, result) {
-                        if (error) {
-                            debug(`Failed to get ${memberDn}:`, error);
-                            return iteratorCallback();
-                        }
-
-                        debug(`Found member object at ${memberDn} adding to group ${group.name}`);
-
-                        const username = result[externalLdapConfig.usernameField];
-                        if (!username) return iteratorCallback();
-
-                        users.getByUsername(username, function (error, result) {
-                            if (error) {
-                                debug(`syncGroupUsers: Failed to get user by username ${username}`, error);
-                                return iteratorCallback();
-                            }
-
-                            groups.addMember(group.id, result.id, function (error) {
-                                if (error && error.reason !== BoxError.ALREADY_EXISTS) debug('syncGroupUsers: Failed to add member', error);
-                                iteratorCallback();
-                            });
-                        });
-                    });
-                }, function (error) {
-                    if (error) debug('syncGroupUsers: ', error);
-                    iteratorCallback();
-                });
-            });
-        }, callback);
-    });
-}
-
 function sync(progressCallback, callback) {
     assert.strictEqual(typeof progressCallback, 'function');
     assert.strictEqual(typeof callback, 'function');
@@ -571,18 +265,58 @@ function sync(progressCallback, callback) {
         if (error) return callback(error);
         if (externalLdapConfig.provider === 'noop') return callback(new BoxError(BoxError.BAD_STATE, 'not enabled'));
 
-        async.series([
-            syncUsers.bind(null, externalLdapConfig, progressCallback),
-            syncGroups.bind(null, externalLdapConfig, progressCallback),
-            syncGroupUsers.bind(null, externalLdapConfig, progressCallback)
-        ], function (error) {
+        ldapSearch(externalLdapConfig, {}, function (error, ldapUsers) {
             if (error) return callback(error);
 
-            progressCallback({ percent: 100, message: 'Done' });
+            debug(`Found ${ldapUsers.length} users`);
+            let percent = 10;
+            let step = 90/(ldapUsers.length+1); // ensure no divide by 0
 
-            debug('sync: ldap sync is done', error);
+            // we ignore all errors here and just log them for now
+            async.eachSeries(ldapUsers, function (user, iteratorCallback) {
+                user = translateUser(externalLdapConfig, user);
 
-            callback(error);
+                if (!validUserRequirements(user)) return iteratorCallback();
+
+                percent += step;
+                progressCallback({ percent, message: `Syncing... ${user.username}` });
+
+                users.getByUsername(user.username, function (error, result) {
+                    if (error && error.reason !== BoxError.NOT_FOUND) {
+                        debug(`Could not find user with username ${user.username}: ${error.message}`);
+                        return iteratorCallback();
+                    }
+
+                    if (error) {
+                        debug(`[adding user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                        users.create(user.username, null /* password */, user.email, user.displayName, { source: 'ldap' }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                            if (error) console.error('Failed to create user', user, error);
+                            iteratorCallback();
+                        });
+                    } else if (result.source !== 'ldap') {
+                        debug(`[conflicting user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                        iteratorCallback();
+                    } else if (result.email !== user.email || result.displayName !== user.displayName) {
+                        debug(`[updating user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                        users.update(result, { email: user.email, fallbackEmail: user.email, displayName: user.displayName }, auditSource.EXTERNAL_LDAP_TASK, function (error) {
+                            if (error) debug('Failed to update user', user, error);
+
+                            iteratorCallback();
+                        });
+                    } else {
+                        // user known and up-to-date
+                        debug(`[up-to-date user] username=${user.username} email=${user.email} displayName=${user.displayName}`);
+
+                        iteratorCallback();
+                    }
+                });
+            }, function (error) {
+                debug('sync: ldap sync is done', error);
+                callback(error);
+            });
         });
     });
 }

src/graphs.js

@@ -5,7 +5,6 @@ exports = module.exports = {
 };
 
 var assert = require('assert'),
-    async = require('async'),
     infra = require('./infra_version.js'),
     paths = require('./paths.js'),
     shell = require('./shell.js');
@@ -27,7 +26,7 @@ function startGraphite(existingInfra, callback) {
                 --log-opt syslog-address=udp://127.0.0.1:2514 \
                 --log-opt syslog-format=rfc5424 \
                 --log-opt tag=graphite \
-                -m 150m \
+                -m 75m \
                 --memory-swap 150m \
                 --dns 172.18.0.1 \
                 --dns-search=. \
@@ -38,9 +37,5 @@ function startGraphite(existingInfra, callback) {
                 --label isCloudronManaged=true \
                 --read-only -v /tmp -v /run "${tag}"`;
 
-    async.series([
-        shell.exec.bind(null, 'stopGraphite', 'docker stop graphite || true'),
-        shell.exec.bind(null, 'removeGraphite', 'docker rm -f graphite || true'),
-        shell.exec.bind(null, 'startGraphite', cmd)
-    ], callback);
+    shell.exec('startGraphite', cmd, callback);
 }

src/groupdb.js

@@ -2,7 +2,6 @@
 
 exports = module.exports = {
     get: get,
-    getByName: getByName,
     getWithMembers: getWithMembers,
     getAll: getAll,
     getAllWithMembers: getAllWithMembers,
@@ -20,6 +19,8 @@ exports = module.exports = {
     getMembership: getMembership,
     setMembership: setMembership,
 
+    getGroups: getGroups,
+
     _clear: clear
 };
 
@@ -27,7 +28,7 @@ var assert = require('assert'),
     BoxError = require('./boxerror.js'),
     database = require('./database.js');
 
-var GROUPS_FIELDS = [ 'id', 'name', 'source' ].join(',');
+var GROUPS_FIELDS = [ 'id', 'name' ].join(',');
 
 function get(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
@@ -41,18 +42,6 @@ function get(groupId, callback) {
     });
 }
 
-function getByName(name, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups WHERE name = ?', [ name ], function (error, result) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-        if (result.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
-
-        callback(null, result[0]);
-    });
-}
-
 function getWithMembers(groupId, callback) {
     assert.strictEqual(typeof groupId, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -74,7 +63,7 @@ function getWithMembers(groupId, callback) {
 function getAll(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups ORDER BY name', function (error, results) {
+    database.query('SELECT ' + GROUPS_FIELDS + ' FROM userGroups', function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
         callback(null, results);
@@ -84,8 +73,9 @@ function getAll(callback) {
 function getAllWithMembers(callback) {
     database.query('SELECT ' + GROUPS_FIELDS + ',GROUP_CONCAT(groupMembers.userId) AS userIds ' +
                     ' FROM userGroups LEFT OUTER JOIN groupMembers ON userGroups.id = groupMembers.groupId ' +
-                    ' GROUP BY userGroups.id ORDER BY name', function (error, results) {
+                    ' GROUP BY userGroups.id', function (error, results) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (results.length === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Group not found'));
 
         results.forEach(function (result) { result.userIds = result.userIds ? result.userIds.split(',') : [ ]; });
 
@@ -93,13 +83,12 @@ function getAllWithMembers(callback) {
     });
 }
 
-function add(id, name, source, callback) {
+function add(id, name, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof source, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('INSERT INTO userGroups (id, name, source) VALUES (?, ?, ?)', [ id, name, source ], function (error, result) {
+    database.query('INSERT INTO userGroups (id, name) VALUES (?, ?)', [ id, name ], function (error, result) {
         if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, error));
         if (error || result.affectedRows !== 1) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
@@ -271,3 +260,15 @@ function isMember(groupId, userId, callback) {
         callback(null, result.length !== 0);
     });
 }
+
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('SELECT ' + GROUPS_FIELDS + ' ' +
+        ' FROM userGroups INNER JOIN groupMembers ON userGroups.id = groupMembers.groupId AND groupMembers.userId = ?', [ userId ], function (error, results) {
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null, results);
+    });
+}

src/groups.js

@@ -4,7 +4,6 @@ exports = module.exports = {
     create: create,
     remove: remove,
     get: get,
-    getByName: getByName,
     update: update,
     getWithMembers: getWithMembers,
     getAll: getAll,
@@ -16,6 +15,8 @@ exports = module.exports = {
     removeMember: removeMember,
     isMember: isMember,
 
+    getGroups: getGroups,
+
     setMembership: setMembership,
     getMembership: getMembership,
 
@@ -44,17 +45,8 @@ function validateGroupname(name) {
     return null;
 }
 
-function validateGroupSource(source) {
-    assert.strictEqual(typeof source, 'string');
-
-    if (source !== '' && source !== 'ldap') return new BoxError(BoxError.BAD_FIELD, 'source must be "" or "ldap"', { field: source });
-
-    return null;
-}
-
-function create(name, source, callback) {
+function create(name, callback) {
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof source, 'string');
     assert.strictEqual(typeof callback, 'function');
 
     // we store names in lowercase
@@ -63,11 +55,8 @@ function create(name, source, callback) {
     var error = validateGroupname(name);
     if (error) return callback(error);
 
-    error = validateGroupSource(source);
-    if (error) return callback(error);
-
     var id = 'gid-' + uuid.v4();
-    groupdb.add(id, name, source, function (error) {
+    groupdb.add(id, name, function (error) {
         if (error) return callback(error);
 
         callback(null, { id: id, name: name });
@@ -96,17 +85,6 @@ function get(id, callback) {
     });
 }
 
-function getByName(name, callback) {
-    assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    groupdb.getByName(name, function (error, result) {
-        if (error) return callback(error);
-
-        return callback(null, result);
-    });
-}
-
 function getWithMembers(id, callback) {
     assert.strictEqual(typeof id, 'string');
     assert.strictEqual(typeof callback, 'function');
@@ -239,6 +217,17 @@ function update(groupId, data, callback) {
     });
 }
 
+function getGroups(userId, callback) {
+    assert.strictEqual(typeof userId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    groupdb.getGroups(userId, function (error, results) {
+        if (error) return callback(error);
+
+        callback(null, results);
+    });
+}
+
 function count(callback) {
     assert.strictEqual(typeof callback, 'function');
 

src/infra_version.js

@@ -9,19 +9,18 @@ exports = module.exports = {
     'version': '48.17.0',
 
     'baseImages': [
-        { repo: 'cloudron/base', tag: 'cloudron/base:2.0.0@sha256:f9fea80513aa7c92fe2e7bf3978b54c8ac5222f47a9a32a7f8833edf0eb5a4f4' }
+        { repo: 'cloudron/base', tag: 'cloudron/base:1.0.0@sha256:147a648a068a2e746644746bbfb42eb7a50d682437cead3c67c933c546357617' }
     ],
 
     // a major version bump in the db containers will trigger the restore logic that uses the db dumps
     // docker inspect --format='{{index .RepoDigests 0}}' $IMAGE to get the sha256
     'images': {
-        'turn': { repo: 'cloudron/turn', tag: 'cloudron/turn:1.1.0@sha256:e1dd22aa6eef5beb7339834b200a8bb787ffc2264ce11139857a054108fefb4f' },
-        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.3.1@sha256:c1145d43c8a912fe6f5a5629a4052454a4aa6f23391c1efbffeec9d12d72a256' },
-        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:3.0.0@sha256:b00e5118a8f829c422234117bf113803be79a1d5102c51497c6d3005b041ce37' },
-        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:3.0.0@sha256:59e50b1f55e433ffdf6d678f8c658812b4119f631db8325572a52ee40d3bc562' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.3.0@sha256:0e31ec817e235b1814c04af97b1e7cf0053384aca2569570ce92bef0d95e94d2' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.9.4@sha256:0e169b97a0584a76197d2bbc039d8698bf93f815588b3b43c251bd83dd545465' },
-        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.3.0@sha256:b7bc1ca4f4d0603a01369a689129aa273a938ce195fe43d00d42f4f2d5212f50' },
-        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:2.0.2@sha256:cbd604eaa970c99ba5c4c2e7984929668e05de824172f880e8c576b2fb7c976d' }
+        'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.1.0@sha256:eee0dfd3829d563f2063084bc0d7c8802c4bdd6e233159c6226a17ff7a9a3503' },
+        'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:2.0.2@sha256:6dcee0731dfb9b013ed94d56205eee219040ee806c7e251db3b3886eaa4947ff' },
+        'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:2.1.0@sha256:6d1bf221cfe6124957e2c58b57c0a47214353496009296acb16adf56df1da9d5' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.1.0@sha256:f2cda21bd15c21bbf44432df412525369ef831a2d53860b5c5b1675e6f384de2' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.6.5@sha256:d17cc0a3d2b6431cb683109abf40fffb91199e2af1d6d99f81d8ec3a1e1bb442' },
+        'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.2.0@sha256:fc9ca69d16e6ebdbd98ed53143d4a0d2212eef60cb638dc71219234e6f427a2c' },
+        'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:0.1.0@sha256:e177c5bf5f38c84ce1dea35649c22a1b05f96eec67a54a812c5a35e585670f0f' }
     }
 };

src/janitor.js

@@ -16,15 +16,21 @@ const NOOP_CALLBACK = function () { };
 
 const gConnection = new Docker({ socketPath: '/var/run/docker.sock' });
 
-function cleanupTokens(callback) {
-    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+function ignoreError(func) {
+    return function (callback) {
+        func(function (error) {
+            if (error) console.error('Ignored error:', error);
 
-    callback = callback || NOOP_CALLBACK;
+            callback();
+        });
+    };
+}
 
-    debug('Cleaning up expired tokens');
+function cleanupExpiredTokens(callback) {
+    assert.strictEqual(typeof callback, 'function');
 
     tokendb.delExpired(function (error, result) {
-        if (error) return debug('cleanupTokens: error removing expired tokens', error);
+        if (error) return callback(error);
 
         debug('Cleaned up %s expired tokens.', result);
 
@@ -32,6 +38,16 @@ function cleanupTokens(callback) {
     });
 }
 
+function cleanupTokens(callback) {
+    assert(!callback || typeof callback === 'function'); // callback is null when called from cronjob
+
+    debug('Cleaning up expired tokens');
+
+    async.series([
+        ignoreError(cleanupExpiredTokens)
+    ], callback);
+}
+
 function cleanupTmpVolume(containerInfo, callback) {
     assert.strictEqual(typeof containerInfo, 'object');
     assert.strictEqual(typeof callback, 'function');

src/ldap.js

@@ -154,6 +154,7 @@ function userSearch(req, res, next) {
                     givenName: firstName,
                     username: user.username,
                     samaccountname: user.username,      // to support ActiveDirectory clients
+                    isadmin: users.compareRoles(user.role, users.ROLE_ADMIN) >= 0,
                     memberof: groups
                 }
             };
@@ -346,7 +347,7 @@ function mailboxSearch(req, res, next) {
                         if (error) return callback(error);
 
                         aliases.forEach(function (a, idx) {
-                            obj.attributes['mail' + idx] = `${a.name}@${a.domain}`;
+                            obj.attributes['mail' + idx] = `${a}@${mailbox.domain}`;
                         });
 
                         // ensure all filter values are also lowercase
@@ -391,7 +392,7 @@ function mailAliasSearch(req, res, next) {
                 objectclass: ['nisMailAlias'],
                 objectcategory: 'nisMailAlias',
                 cn: `${alias.name}@${alias.domain}`,
-                rfc822MailMember: `${alias.aliasName}@${alias.aliasDomain}`
+                rfc822MailMember: `${alias.aliasTarget}@${alias.domain}`
             }
         };
 
@@ -417,7 +418,7 @@ function mailingListSearch(req, res, next) {
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
     const name = parts[0], domain = parts[1];
 
-    mail.resolveList(parts[0], parts[1], function (error, resolvedMembers, list) {
+    mail.resolveList(parts[0], parts[1], function (error, resolvedMembers) {
         if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
         if (error) return next(new ldap.OperationsError(error.toString()));
 
@@ -430,7 +431,6 @@ function mailingListSearch(req, res, next) {
                 objectcategory: 'mailGroup',
                 cn: `${name}@${domain}`, // fully qualified
                 mail: `${name}@${domain}`,
-                membersOnly: list.membersOnly, // ldapjs only supports strings and string array. so this is not a bool!
                 mgrpRFC822MailMember: resolvedMembers // fully qualified
             }
         };
@@ -534,16 +534,13 @@ function authenticateSftp(req, res, next) {
     var parts = email.split('@');
     if (parts.length !== 2) return next(new ldap.NoSuchObjectError(req.dn.toString()));
 
-    apps.getByFqdn(parts[1], function (error, app) {
+    // actual user bind
+    users.verifyWithUsername(parts[0], req.credentials, users.AP_SFTP, function (error) {
         if (error) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
 
-        users.verifyWithUsername(parts[0], req.credentials, app.id, function (error) {
-            if (error) return next(new ldap.InvalidCredentialsError(req.dn.toString()));
+        debug('sftp auth: success');
 
-            debug('sftp auth: success');
-
-            res.end();
-        });
+        res.end();
     });
 }
 
@@ -577,7 +574,7 @@ function userSearchSftp(req, res, next) {
                 var obj = {
                     dn: ldap.parseDN(`cn=${username}@${appFqdn},ou=sftp,dc=cloudron`).toString(),
                     attributes: {
-                        homeDirectory: path.join('/app/data', app.id),
+                        homeDirectory: path.join('/app/data', app.id, 'data'),
                         objectclass: ['user'],
                         objectcategory: 'person',
                         cn: user.id,
@@ -618,7 +615,10 @@ function authenticateMailAddon(req, res, next) {
         // note: with sendmail addon, apps can send mail without a mailbox (unlike users)
         appdb.getAppIdByAddonConfigValue(addonId, namePattern, req.credentials || '', function (error, appId) {
             if (error && error.reason !== BoxError.NOT_FOUND) return next(new ldap.OperationsError(error.message));
-            if (appId) return res.end();
+            if (appId) { // matched app password
+                eventlog.add(eventlog.ACTION_APP_LOGIN, { authType: 'ldap', mailboxId: email }, { appId: appId, addonId: addonId });
+                return res.end();
+            }
 
             mailboxdb.getMailbox(parts[0], parts[1], function (error, mailbox) {
                 if (error && error.reason === BoxError.NOT_FOUND) return next(new ldap.NoSuchObjectError(req.dn.toString()));
@@ -645,14 +645,14 @@ function start(callback) {
         debug: NOOP,
         info: debug,
         warn: debug,
-        error: debug,
-        fatal: debug
+        error: console.error,
+        fatal: console.error
     };
 
     gServer = ldap.createServer({ log: logger });
 
     gServer.on('error', function (error) {
-        debug('start: server error ', error);
+        console.error('LDAP:', error);
     });
 
     gServer.search('ou=users,dc=cloudron', authenticateApp, userSearch);

src/logrotate.ejs

@@ -8,7 +8,6 @@
     maxsize 1M
     missingok
     delaycompress
-    # this truncates the original log file and not the rotated one
     copytruncate
 }
 
@@ -19,7 +18,6 @@
     missingok
     # we never compress so we can simply tail the files
     nocompress
-    # this truncates the original log file and not the rotated one
     copytruncate
 }
 

src/mail.js

@@ -1,54 +1,53 @@
 'use strict';
 
 exports = module.exports = {
-    getStatus,
-    checkConfiguration,
+    getStatus: getStatus,
+    checkConfiguration: checkConfiguration,
 
-    getDomains,
+    getDomains: getDomains,
 
-    getDomain,
-    clearDomains,
+    getDomain: getDomain,
+    addDomain: addDomain,
+    removeDomain: removeDomain,
+    clearDomains: clearDomains,
 
-    onDomainAdded,
-    onDomainRemoved,
-    onMailFqdnChanged,
+    removePrivateFields: removePrivateFields,
 
-    removePrivateFields,
+    setDnsRecords: setDnsRecords,
+    onMailFqdnChanged: onMailFqdnChanged,
 
-    setDnsRecords,
+    validateName: validateName,
 
-    validateName,
-
-    setMailFromValidation,
-    setCatchAllAddress,
-    setMailRelay,
-    setMailEnabled,
+    setMailFromValidation: setMailFromValidation,
+    setCatchAllAddress: setCatchAllAddress,
+    setMailRelay: setMailRelay,
+    setMailEnabled: setMailEnabled,
 
     startMail: restartMail,
-    restartMail,
-    handleCertChanged,
-    getMailAuth,
+    restartMail: restartMail,
+    handleCertChanged: handleCertChanged,
+    getMailAuth: getMailAuth,
 
-    sendTestMail,
+    sendTestMail: sendTestMail,
 
-    getMailboxCount,
-    listMailboxes,
-    getMailbox,
-    addMailbox,
-    updateMailboxOwner,
-    removeMailbox,
+    listMailboxes: listMailboxes,
+    removeMailboxes: removeMailboxes,
+    getMailbox: getMailbox,
+    addMailbox: addMailbox,
+    updateMailboxOwner: updateMailboxOwner,
+    removeMailbox: removeMailbox,
 
-    getAliases,
-    setAliases,
+    listAliases: listAliases,
+    getAliases: getAliases,
+    setAliases: setAliases,
 
-    getLists,
-    getList,
-    addList,
-    updateList,
-    removeList,
-    resolveList,
+    getLists: getLists,
+    getList: getList,
+    addList: addList,
+    updateList: updateList,
+    removeList: removeList,
+    resolveList: resolveList,
 
-    _removeMailboxes: removeMailboxes,
     _readDkimPublicKeySync: readDkimPublicKeySync
 };
 
@@ -82,7 +81,6 @@ var assert = require('assert'),
 
 const DNS_OPTIONS = { timeout: 5000 };
 var NOOP_CALLBACK = function (error) { if (error) debug(error); };
-const REMOVE_MAILBOX = path.join(__dirname, 'scripts/rmmailbox.sh');
 
 function validateName(name) {
     assert.strictEqual(typeof name, 'string');
@@ -102,6 +100,7 @@ function checkOutboundPort25(callback) {
     var smtpServer = _.sample([
         'smtp.gmail.com',
         'smtp.live.com',
+        'smtp.mail.yahoo.com',
         'smtp.1und1.de',
     ]);
 
@@ -208,8 +207,7 @@ function checkDkim(mailDomain, callback) {
 
         if (txtRecords.length !== 0) {
             dkim.value = txtRecords[0].join('');
-            const actual = txtToDict(dkim.value);
-            dkim.status = actual.p === dkimKey;
+            dkim.status = (dkim.value === dkim.expected);
         }
 
         callback(null, dkim);
@@ -270,7 +268,7 @@ function checkMx(domain, mailFqdn, callback) {
         if (error) return callback(error, mx);
         if (mxRecords.length === 0) return callback(null, mx);
 
-        mx.status = mxRecords.some(mx => mx.exchange === mailFqdn); // this lets use change priority and/or setup backup MX
+        mx.status = mxRecords.length == 1 && mxRecords[0].exchange === mailFqdn;
         mx.value = mxRecords.map(function (r) { return r.priority + ' ' + r.exchange + '.'; }).join(' ');
 
         if (mx.status) return callback(null, mx); // MX record is "my."
@@ -314,8 +312,9 @@ function checkDmarc(domain, callback) {
 
         if (txtRecords.length !== 0) {
             dmarc.value = txtRecords[0].join('');
-            const actual = txtToDict(dmarc.value);
-            dmarc.status = actual.v === 'DMARC1'; // see box#666
+            // allow extra fields in dmarc like rua
+            const actual = txtToDict(dmarc.value), expected = txtToDict(dmarc.expected);
+            dmarc.status = Object.keys(expected).every(k => expected[k] === actual[k]);
         }
 
         callback(null, dmarc);
@@ -630,10 +629,7 @@ function configureMail(mailFqdn, mailDomain, callback) {
         if (!safe.child_process.execSync(`cp ${bundle.certFilePath} ${mailCertFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create cert file:' + safe.error.message));
         if (!safe.child_process.execSync(`cp ${bundle.keyFilePath} ${mailKeyFilePath}`)) return callback(new BoxError(BoxError.FS_ERROR, 'Could not create key file:' + safe.error.message));
 
-        async.series([
-            shell.exec.bind(null, 'stopMail', 'docker stop mail || true'),
-            shell.exec.bind(null, 'removeMail', 'docker rm -f mail || true'),
-        ], function (error) {
+        shell.exec('startMail', 'docker rm -f mail || true', function (error) {
             if (error) return callback(error);
 
             createMailConfig(mailFqdn, mailDomain, function (error, allowInbound) {
@@ -802,7 +798,6 @@ function ensureDkimKeySync(mailDomain) {
         return new BoxError(BoxError.FS_ERROR, safe.error);
     }
 
-    // https://www.unlocktheinbox.com/dkim-key-length-statistics/ and https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-authentication-dkim-easy.html for key size
     if (!safe.child_process.execSync('openssl genrsa -out ' + dkimPrivateKeyFile + ' 1024')) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);
     if (!safe.child_process.execSync('openssl rsa -in ' + dkimPrivateKeyFile + ' -out ' + dkimPublicKeyFile + ' -pubout -outform PEM')) return new BoxError(BoxError.OPENSSL_ERROR, safe.error);
 
@@ -910,21 +905,37 @@ function onMailFqdnChanged(callback) {
     });
 }
 
-function onDomainAdded(domain, callback) {
+function addDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    async.series([
-        upsertDnsRecords.bind(null, domain, settings.mailFqdn()), // do this first to ensure DKIM keys
-        restartMailIfActivated
-    ], callback);
+    const dkimSelector = domain === settings.adminDomain() ? 'cloudron' : ('cloudron-' + settings.adminDomain().replace(/\./g, ''));
+
+    maildb.add(domain, { dkimSelector }, function (error) {
+        if (error) return callback(error);
+
+        async.series([
+            upsertDnsRecords.bind(null, domain, settings.mailFqdn()), // do this first to ensure DKIM keys
+            restartMailIfActivated
+        ], NOOP_CALLBACK); // do these asynchronously
+
+        callback();
+    });
 }
 
-function onDomainRemoved(domain, callback) {
+function removeDomain(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    restartMail(callback);
+    if (domain === settings.adminDomain()) return callback(new BoxError(BoxError.CONFLICT));
+
+    maildb.del(domain, function (error) {
+        if (error) return callback(error);
+
+        restartMail(NOOP_CALLBACK);
+
+        callback();
+    });
 }
 
 function clearDomains(callback) {
@@ -1036,25 +1047,13 @@ function sendTestMail(domain, to, callback) {
     });
 }
 
-function listMailboxes(domain, search, page, perPage, callback) {
+function listMailboxes(domain, page, perPage, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert(typeof search === 'string' || search === null);
     assert.strictEqual(typeof page, 'number');
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    mailboxdb.listMailboxes(domain, search, page, perPage, function (error, result) {
-        if (error) return callback(error);
-
-        callback(null, result);
-    });
-}
-
-function getMailboxCount(domain, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    mailboxdb.getMailboxCount(domain, function (error, result) {
+    mailboxdb.listMailboxes(domain, page, perPage, function (error, result) {
         if (error) return callback(error);
 
         callback(null, result);
@@ -1127,25 +1126,31 @@ function updateMailboxOwner(name, domain, userId, auditSource, callback) {
     });
 }
 
-function removeMailbox(name, domain, options, auditSource, callback) {
+function removeMailbox(name, domain, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof name, 'string');
-    assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    const deleteMailFunc = options.deleteMails ? shell.sudo.bind(null, 'removeMailbox', [ REMOVE_MAILBOX, `${name}@${domain}` ], {}) : (next) => next();
+    mailboxdb.del(name, domain, function (error) {
+        if (error) return callback(error);
 
-    deleteMailFunc(function (error) {
-        if (error) return callback(new BoxError(BoxError.FS_ERROR, `Error removing mailbox: ${error.message}`));
+        eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
 
-        mailboxdb.del(name, domain, function (error) {
-            if (error) return callback(error);
+        callback(null);
+    });
+}
 
-            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_REMOVE, auditSource, { name, domain });
+function listAliases(domain, page, perPage, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
 
-            callback();
-        });
+    mailboxdb.listAliases(domain, page, perPage, function (error, result) {
+        if (error) return callback(error);
+
+        callback(null, result);
     });
 }
 
@@ -1172,15 +1177,12 @@ function setAliases(name, domain, aliases, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     for (var i = 0; i < aliases.length; i++) {
-        let name = aliases[i].name.toLowerCase();
-        let domain = aliases[i].domain.toLowerCase();
+        aliases[i] = aliases[i].toLowerCase();
 
-        let error = validateName(name);
+        var error = validateName(aliases[i]);
         if (error) return callback(error);
-
-        if (!validator.isEmail(`${name}@${domain}`)) return callback(new BoxError(BoxError.BAD_FIELD, `Invalid email: ${name}@${domain}`));
-        aliases[i] = { name, domain };
     }
+
     mailboxdb.setAliasesForName(name, domain, aliases, function (error) {
         if (error) return callback(error);
 
@@ -1188,14 +1190,11 @@ function setAliases(name, domain, aliases, callback) {
     });
 }
 
-function getLists(domain, search, page, perPage, callback) {
+function getLists(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert(typeof search === 'string' || search === null);
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    mailboxdb.getLists(domain, search, page, perPage, function (error, result) {
+    mailboxdb.getLists(domain, function (error, result) {
         if (error) return callback(error);
 
         callback(null, result);
@@ -1214,11 +1213,10 @@ function getList(name, domain, callback) {
     });
 }
 
-function addList(name, domain, members, membersOnly, auditSource, callback) {
+function addList(name, domain, members, auditSource, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof name, 'string');
     assert(Array.isArray(members));
-    assert.strictEqual(typeof membersOnly, 'boolean');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1231,20 +1229,19 @@ function addList(name, domain, members, membersOnly, auditSource, callback) {
         if (!validator.isEmail(members[i])) return callback(new BoxError(BoxError.BAD_FIELD, 'Invalid mail member: ' + members[i]));
     }
 
-    mailboxdb.addList(name, domain, members, membersOnly, function (error) {
+    mailboxdb.addList(name, domain, members, function (error) {
         if (error) return callback(error);
 
-        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members, membersOnly });
+        eventlog.add(eventlog.ACTION_MAIL_LIST_ADD, auditSource, { name, domain, members });
 
         callback();
     });
 }
 
-function updateList(name, domain, members, membersOnly, auditSource, callback) {
+function updateList(name, domain, members, auditSource, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(members));
-    assert.strictEqual(typeof membersOnly, 'boolean');
     assert.strictEqual(typeof auditSource, 'object');
     assert.strictEqual(typeof callback, 'function');
 
@@ -1260,10 +1257,10 @@ function updateList(name, domain, members, membersOnly, auditSource, callback) {
     getList(name, domain, function (error, result) {
         if (error) return callback(error);
 
-        mailboxdb.updateList(name, domain, members, membersOnly, function (error) {
+        mailboxdb.updateList(name, domain, members, function (error) {
             if (error) return callback(error);
 
-            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members, membersOnly });
+            eventlog.add(eventlog.ACTION_MAIL_MAILBOX_UPDATE, auditSource, { name, domain, oldMembers: result.members, members });
 
             callback(null);
         });
@@ -1285,7 +1282,6 @@ function removeList(name, domain, auditSource, callback) {
     });
 }
 
-// resolves the members of a list. i.e the lists and aliases
 function resolveList(listName, listDomain, callback) {
     assert.strictEqual(typeof listName, 'string');
     assert.strictEqual(typeof listDomain, 'string');
@@ -1316,21 +1312,18 @@ function resolveList(listName, listDomain, callback) {
                 visited.push(member);
 
                 mailboxdb.get(memberName, memberDomain, function (error, entry) {
-                    if (error && error.reason == BoxError.NOT_FOUND) { result.push(member); return iteratorCallback(); } // let it bounce
+                    if (error && error.reason == BoxError.NOT_FOUND) { result.push(member); return iteratorCallback(); }
                     if (error) return iteratorCallback(error);
 
-                    if (entry.type === mailboxdb.TYPE_MAILBOX) { // concrete mailbox
-                        result.push(member);
-                    } else if (entry.type === mailboxdb.TYPE_ALIAS) { // resolve aliases
-                        toResolve = toResolve.concat(`${entry.aliasName}@${entry.aliasDomain}`);
-                    } else { // resolve list members
-                        toResolve = toResolve.concat(entry.members);
-                    }
+                    if (entry.type === mailboxdb.TYPE_MAILBOX) { result.push(member); return iteratorCallback(); }
+                    // no need to resolve alias because we only allow one level and within same domain
+                    if (entry.type === mailboxdb.TYPE_ALIAS) { result.push(`${entry.aliasTarget}@${entry.domain}`); return iteratorCallback(); }
 
+                    toResolve = toResolve.concat(entry.members);
                     iteratorCallback();
                 });
             }, function (error) {
-                callback(error, result, list);
+                callback(error, result);
             });
         });
     });

src/mail_templates/oom_event.ejs

@@ -6,7 +6,7 @@ Dear <%= cloudronName %> Admin,
 
 If this message appears repeatedly, give the app more memory.
 
-* To increase an app's memory limit - https://cloudron.io/documentation/apps/#memory-limit
+* To increase an app's memory limit - https://cloudron.io/documentation/apps/#increasing-the-memory-limit-of-an-app
 * To increase a service's memory limit - https://cloudron.io/documentation/troubleshooting/#services
 
 Out of memory event:

src/mail_templates/password_reset.ejs

@@ -8,7 +8,7 @@ be reset. If you did not request this reset, please ignore this message.
 To reset your password, please visit the following page:
 <%- resetLink %>
 
-Please note that the password reset link will expire in 24 hours.
+
 
 Powered by https://cloudron.io
 
@@ -29,10 +29,6 @@ Powered by https://cloudron.io
     <a href="<%= resetLink %>">Click to reset your password</a>
 </p>
 
-<br/>
-
-Please note that the password reset link will expire in 24 hours.
-
 <br/>
 <br/>
 

src/mail_templates/welcome_user.ejs

@@ -11,7 +11,6 @@ Follow the link to get started.
 You are receiving this email because you were invited by <%= invitor.email %>.
 <% } %>
 
-Please note that the invite link will expire in 7 days.
 
 Powered by https://cloudron.io
 
@@ -37,9 +36,6 @@ Powered by https://cloudron.io
         You are receiving this email because you were invited by <%= invitor.email %>.
     <% } %>
 
-    <br/>
-
-        Please note that the invite link will expire in 7 days.
     <br/>
 
     Powered by <a href="https://cloudron.io">Cloudron</a>

src/mailboxdb.js

@@ -1,32 +1,32 @@
 'use strict';
 
 exports = module.exports = {
-    addMailbox,
-    addList,
+    addMailbox: addMailbox,
+    addList: addList,
 
-    updateMailboxOwner,
-    updateList,
-    del,
+    updateMailboxOwner: updateMailboxOwner,
+    updateList: updateList,
+    del: del,
 
-    getMailboxCount,
-    listMailboxes,
-    getLists,
+    listAliases: listAliases,
+    listMailboxes: listMailboxes,
+    getLists: getLists,
 
-    listAllMailboxes,
+    listAllMailboxes: listAllMailboxes,
 
-    get,
-    getMailbox,
-    getList,
-    getAlias,
+    get: get,
+    getMailbox: getMailbox,
+    getList: getList,
+    getAlias: getAlias,
 
-    getAliasesForName,
-    setAliasesForName,
+    getAliasesForName: getAliasesForName,
+    setAliasesForName: setAliasesForName,
 
-    getByOwnerId,
-    delByOwnerId,
-    delByDomain,
+    getByOwnerId: getByOwnerId,
+    delByOwnerId: delByOwnerId,
+    delByDomain: delByDomain,
 
-    updateName,
+    updateName: updateName,
 
     _clear: clear,
 
@@ -38,18 +38,15 @@ exports = module.exports = {
 var assert = require('assert'),
     BoxError = require('./boxerror.js'),
     database = require('./database.js'),
-    mysql = require('mysql'),
     safe = require('safetydance'),
     util = require('util');
 
-var MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'aliasName', 'aliasDomain', 'creationTime', 'membersJson', 'membersOnly', 'domain' ].join(',');
+var MAILBOX_FIELDS = [ 'name', 'type', 'ownerId', 'aliasTarget', 'creationTime', 'membersJson', 'domain' ].join(',');
 
 function postProcess(data) {
     data.members = safe.JSON.parse(data.membersJson) || [ ];
     delete data.membersJson;
 
-    data.membersOnly = !!data.membersOnly;
-
     return data;
 }
 
@@ -81,15 +78,14 @@ function updateMailboxOwner(name, domain, ownerId, callback) {
     });
 }
 
-function addList(name, domain, members, membersOnly, callback) {
+function addList(name, domain, members, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(members));
-    assert.strictEqual(typeof membersOnly, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, membersJson, membersOnly) VALUES (?, ?, ?, ?, ?, ?)',
-        [ name, exports.TYPE_LIST, domain, 'admin', JSON.stringify(members), membersOnly ], function (error) {
+    database.query('INSERT INTO mailboxes (name, type, domain, ownerId, membersJson) VALUES (?, ?, ?, ?, ?)',
+        [ name, exports.TYPE_LIST, domain, 'admin', JSON.stringify(members) ], function (error) {
             if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mailbox already exists'));
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
@@ -97,15 +93,14 @@ function addList(name, domain, members, membersOnly, callback) {
         });
 }
 
-function updateList(name, domain, members, membersOnly, callback) {
+function updateList(name, domain, members, callback) {
     assert.strictEqual(typeof name, 'string');
     assert.strictEqual(typeof domain, 'string');
     assert(Array.isArray(members));
-    assert.strictEqual(typeof membersOnly, 'boolean');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('UPDATE mailboxes SET membersJson = ?, membersOnly = ? WHERE name = ? AND domain = ?',
-        [ JSON.stringify(members), membersOnly, name, domain ], function (error, result) {
+    database.query('UPDATE mailboxes SET membersJson = ? WHERE name = ? AND domain = ?',
+        [ JSON.stringify(members), name, domain ], function (error, result) {
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
             if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
 
@@ -128,7 +123,7 @@ function del(name, domain, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     // deletes aliases as well
-    database.query('DELETE FROM mailboxes WHERE ((name=? AND domain=?) OR (aliasName = ? AND aliasDomain=?))', [ name, domain, name, domain ], function (error, result) {
+    database.query('DELETE FROM mailboxes WHERE (name=? OR aliasTarget = ?) AND domain = ?', [ name, name, domain ], function (error, result) {
         if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
         if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mailbox not found'));
 
@@ -205,44 +200,14 @@ function getMailbox(name, domain, callback) {
         });
 }
 
-function getMailboxCount(domain, callback) {
+function listMailboxes(domain, page, perPage, callback) {
     assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query('SELECT COUNT(*) AS total FROM mailboxes WHERE type = ? AND domain = ?', [ exports.TYPE_MAILBOX, domain ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        callback(null, results[0].total);
-    });
-}
-
-function listMailboxes(domain, search, page, perPage, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert(typeof search === 'string' || search === null);
     assert.strictEqual(typeof page, 'number');
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
-    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ')';
-    query += 'ORDER BY name LIMIT ?,?';
-
-    database.query(query, [ exports.TYPE_MAILBOX, domain, (page-1)*perPage, perPage ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
-
-        results.forEach(function (result) { postProcess(result); });
-
-        callback(null, results);
-    });
-}
-
-function listAllMailboxes(page, perPage, callback) {
-    assert.strictEqual(typeof page, 'number');
-    assert.strictEqual(typeof perPage, 'number');
-    assert.strictEqual(typeof callback, 'function');
-
-    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ?,?`,
-        [ exports.TYPE_MAILBOX, (page-1)*perPage, perPage ], function (error, results) {
+    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ? ORDER BY name LIMIT ${(page-1)*perPage},${perPage}`,
+        [ exports.TYPE_MAILBOX, domain ], function (error, results) {
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
             results.forEach(function (result) { postProcess(result); });
@@ -251,25 +216,33 @@ function listAllMailboxes(page, perPage, callback) {
         });
 }
 
-function getLists(domain, search, page, perPage, callback) {
-    assert.strictEqual(typeof domain, 'string');
-    assert(typeof search === 'string' || search === null);
+function listAllMailboxes(page, perPage, callback) {
     assert.strictEqual(typeof page, 'number');
     assert.strictEqual(typeof perPage, 'number');
     assert.strictEqual(typeof callback, 'function');
 
-    let query = `SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? AND domain = ?`;
-    if (search) query += ' AND (name LIKE ' + mysql.escape('%' + search + '%') + ' OR membersJson LIKE ' + mysql.escape('%' + search + '%') + ')';
+    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE type = ? ORDER BY name LIMIT ${(page-1)*perPage},${perPage}`,
+        [ exports.TYPE_MAILBOX ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
-    query += 'ORDER BY name LIMIT ?,?';
+            results.forEach(function (result) { postProcess(result); });
 
-    database.query(query, [ exports.TYPE_LIST, domain, (page-1)*perPage, perPage ], function (error, results) {
-        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+            callback(null, results);
+        });
+}
 
-        results.forEach(function (result) { postProcess(result); });
+function getLists(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
 
-        callback(null, results);
-    });
+    database.query('SELECT ' + MAILBOX_FIELDS + ' FROM mailboxes WHERE type = ? AND domain = ?',
+        [ exports.TYPE_LIST, domain ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
+            callback(null, results);
+        });
 }
 
 function getList(name, domain, callback) {
@@ -312,10 +285,10 @@ function setAliasesForName(name, domain, aliases, callback) {
 
         var queries = [];
         // clear existing aliases
-        queries.push({ query: 'DELETE FROM mailboxes WHERE aliasName = ? AND aliasDomain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
+        queries.push({ query: 'DELETE FROM mailboxes WHERE aliasTarget = ? AND domain = ? AND type = ?', args: [ name, domain, exports.TYPE_ALIAS ] });
         aliases.forEach(function (alias) {
-            queries.push({ query: 'INSERT INTO mailboxes (name, domain, type, aliasName, aliasDomain, ownerId) VALUES (?, ?, ?, ?, ?, ?)',
-                args: [ alias.name, alias.domain, exports.TYPE_ALIAS, name, domain, results[0].ownerId ] });
+            queries.push({ query: 'INSERT INTO mailboxes (name, type, domain, aliasTarget, ownerId) VALUES (?, ?, ?, ?, ?)',
+                args: [ alias, exports.TYPE_ALIAS, domain, name, results[0].ownerId ] });
         });
 
         database.transaction(queries, function (error) {
@@ -338,10 +311,27 @@ function getAliasesForName(name, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    database.query('SELECT name, domain FROM mailboxes WHERE type = ? AND aliasName = ? AND aliasDomain = ? ORDER BY name',
+    database.query('SELECT name FROM mailboxes WHERE type = ? AND aliasTarget = ? AND domain = ? ORDER BY name',
         [ exports.TYPE_ALIAS, name, domain ], function (error, results) {
             if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
 
+            results = results.map(function (r) { return r.name; });
+            callback(null, results);
+        });
+}
+
+function listAliases(domain, page, perPage, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof page, 'number');
+    assert.strictEqual(typeof perPage, 'number');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query(`SELECT ${MAILBOX_FIELDS} FROM mailboxes WHERE domain = ? AND type = ? ORDER BY name LIMIT ${(page-1)*perPage},${perPage}`,
+        [ domain, exports.TYPE_ALIAS ], function (error, results) {
+            if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+            results.forEach(function (result) { postProcess(result); });
+
             callback(null, results);
         });
 }

src/maildb.js

@@ -1,6 +1,8 @@
 'use strict';
 
 exports = module.exports = {
+    add: add,
+    del: del,
     get: get,
     list: list,
     update: update,
@@ -32,6 +34,20 @@ function postProcess(data) {
     return data;
 }
 
+function add(domain, data, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof data, 'object');
+    assert.strictEqual(typeof callback, 'function');
+
+    database.query('INSERT INTO mail (domain, dkimSelector) VALUES (?, ?)', [ domain, data.dkimSelector || 'cloudron' ], function (error) {
+        if (error && error.code === 'ER_DUP_ENTRY') return callback(new BoxError(BoxError.ALREADY_EXISTS, 'mail domain already exists'));
+        if (error && error.code === 'ER_NO_REFERENCED_ROW_2') return callback(new BoxError(BoxError.NOT_FOUND), 'no such domain');
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+
+        callback(null);
+    });
+}
+
 function clear(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -42,6 +58,20 @@ function clear(callback) {
     });
 }
 
+function del(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    // deletes aliases as well
+    database.query('DELETE FROM mail WHERE domain=?', [ domain ], function (error, result) {
+        if (error && error.code === 'ER_ROW_IS_REFERENCED_2') return callback(new BoxError(BoxError.CONFLICT));
+        if (error) return callback(new BoxError(BoxError.DATABASE_ERROR, error));
+        if (result.affectedRows === 0) return callback(new BoxError(BoxError.NOT_FOUND, 'Mail domain not found'));
+
+        callback(null);
+    });
+}
+
 function get(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');

src/nginxconfig.ejs

@@ -7,7 +7,6 @@ map $http_upgrade $connection_upgrade {
 # http server
 server {
     listen       80;
-    server_tokens off; # hide version
 <% if (hasIPv6) { -%>
     listen       [::]:80;
 <% } -%>
@@ -43,33 +42,33 @@ server {
 server {
 <% if (vhost) { -%>
     server_name  <%= vhost %>;
-    listen       443 ssl http2;
+    listen       443 http2;
 <% if (hasIPv6) { -%>
-    listen       [::]:443 ssl http2;
+    listen       [::]:443 http2;
 <% } -%>
 <% } else { -%>
-    listen       443 ssl http2 default_server;
+    listen       443 http2 default_server;
 <% if (hasIPv6) { -%>
-    listen       [::]:443 ssl http2 default_server;
+    listen       [::]:443 http2 default_server;
 <% } -%>
 <% } -%>
 
-    server_tokens off; # hide version
-
+    ssl                  on;
     # paths are relative to prefix and not to this file
     ssl_certificate      <%= certFilePath %>;
     ssl_certificate_key  <%= keyFilePath %>;
     ssl_session_timeout  5m;
-    ssl_session_cache shared:MozSSL:10m;  # about 40000 sessions
-    ssl_session_tickets off;
+    ssl_session_cache shared:SSL:50m;
 
+    # https://bettercrypto.org/static/applied-crypto-hardening.pdf
+    # https://mozilla.github.io/server-side-tls/ssl-config-generator/
+    # https://cipherli.st/
     # https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html
-    # https://github.com/ssllabs/research/wiki/SSL-and-TLS-Deployment-Best-Practices#25-use-forward-secrecy
-    # ciphers according to https://ssl-config.mozilla.org/#server=nginx&version=1.14.0&config=intermediate&openssl=1.1.1&guideline=5.4
-    ssl_protocols TLSv1.2 TLSv1.3;
-    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA256;
-    ssl_prefer_server_ciphers off;
+    ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # don't use SSLv3 ref: POODLE
 
+    # ciphers according to https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=nginx-1.10.3&openssl=1.0.2g&hsts=yes&profile=modern
+    ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256';
     ssl_dhparam /home/yellowtent/boxdata/dhparams.pem;
     add_header Strict-Transport-Security "max-age=15768000";
 
@@ -137,21 +136,8 @@ server {
         # internal means this is for internal routing and cannot be accessed as URL from browser
         internal;
     }
-
-    location @wellknown-upstream {
-<% if ( endpoint === 'admin' ) { %>
-        proxy_pass http://127.0.0.1:3000;
-<% } else if ( endpoint === 'app' ) { %>
-        proxy_pass http://127.0.0.1:<%= port %>;
-<% } else if ( endpoint === 'redirect' ) { %>
-        return 302 https://<%= redirectTo %>$request_uri;
-<% } %>
-    }
-
-    # user defined .well-known resources
-    location ~ ^/.well-known/(.*)$ {
-        root /home/yellowtent/boxdata/well-known/$host;
-        try_files /$1 @wellknown-upstream;
+    location /appstatus.html {
+        internal;
     }
 
     location / {
@@ -195,11 +181,6 @@ server {
             client_max_body_size 0;
         }
 
-        location ~ ^/api/v1/apps/.*/files/ {
-            proxy_pass   http://127.0.0.1:3000;
-            client_max_body_size 0;
-        }
-
         # graphite paths (uncomment block below and visit /graphite-web/dashboard)
         # remember to comment out the CSP policy as well to access the graphite dashboard
         # location ~ ^/graphite-web/ {

src/notifications.js

@@ -156,7 +156,7 @@ function oomEvent(eventId, app, addon, containerId, event, callback) {
     if (app) {
         program = `App ${app.fqdn}`;
         title = `The application ${app.fqdn} (${app.manifest.title}) ran out of memory.`;
-        message = 'The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](https://cloudron.io/documentation/apps/#memory-limit)';
+        message = 'The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](https://cloudron.io/documentation/apps/#increasing-the-memory-limit-of-an-app)';
     } else if (addon) {
         program = `${addon.name} service`;
         title = `The ${addon.name} service ran out of memory`;
@@ -211,7 +211,7 @@ function appUpdated(eventId, app, callback) {
             if (error) return callback(error);
 
             mailer.appUpdated(admin.email, app, function (error) {
-                if (error) debug('appUpdated: Failed to send app updated email', error); // non fatal
+                if (error) console.error('Failed to send app updated email', error); // non fatal
                 done();
             });
         });
@@ -273,7 +273,7 @@ function alert(id, title, message, callback) {
     assert.strictEqual(typeof message, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`alert: id=${id} title=${title}`);
+    debug(`alert: id=${id} title=${title} message=${message}`);
 
     const acknowledged = !message;
 
@@ -301,7 +301,7 @@ function alert(id, title, message, callback) {
             });
         });
     }, function (error) {
-        if (error) debug('alert: error notifying', error);
+        if (error) console.error(error);
 
         callback();
     });

src/paths.js

@@ -17,11 +17,12 @@ exports = module.exports = {
     CLOUDRON_DEFAULT_AVATAR_FILE: path.join(__dirname + '/../assets/avatar.png'),
     INFRA_VERSION_FILE: path.join(baseDir(), 'platformdata/INFRA_VERSION'),
 
+    LICENSE_FILE: '/etc/cloudron/LICENSE',
     PROVIDER_FILE: '/etc/cloudron/PROVIDER',
 
     PLATFORM_DATA_DIR: path.join(baseDir(), 'platformdata'),
     APPS_DATA_DIR: path.join(baseDir(), 'appsdata'),
-    BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),  // box data dir is part of box backup
+    BOX_DATA_DIR: path.join(baseDir(), 'boxdata'),
 
     ACME_CHALLENGES_DIR: path.join(baseDir(), 'platformdata/acme'),
     ADDON_CONFIG_DIR: path.join(baseDir(), 'platformdata/addons'),
@@ -45,12 +46,10 @@ exports = module.exports = {
     APP_CERTS_DIR: path.join(baseDir(), 'boxdata/certs'),
     CLOUDRON_AVATAR_FILE: path.join(baseDir(), 'boxdata/avatar.png'),
     UPDATE_CHECKER_FILE: path.join(baseDir(), 'boxdata/updatechecker.json'),
-    ADDON_TURN_SECRET_FILE: path.join(baseDir(), 'boxdata/addon-turn-secret'),
 
     LOG_DIR: path.join(baseDir(), 'platformdata/logs'),
     TASKS_LOG_DIR: path.join(baseDir(), 'platformdata/logs/tasks'),
     CRASH_LOG_DIR: path.join(baseDir(), 'platformdata/logs/crash'),
-    BOX_LOG_FILE: path.join(baseDir(), 'platformdata/logs/box.log'),
 
     GHOST_USER_FILE: path.join(baseDir(), 'platformdata/cloudron_ghost.json'),
 

src/platform.js

@@ -2,7 +2,7 @@
 
 exports = module.exports = {
     start: start,
-    stopAllTasks: stopAllTasks,
+    stop: stop,
 
     // exported for testing
     _isReady: false
@@ -56,8 +56,9 @@ function start(callback) {
     if (error) return callback(error);
 
     async.series([
-        (next) => { if (existingInfra.version !== infra.version) removeAllContainers(existingInfra, next); else next(); },
-        markApps.bind(null, existingInfra), // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
+        stopContainers.bind(null, existingInfra),
+        // mark app state before we start addons. this gives the db import logic a chance to mark an app as errored
+        startApps.bind(null, existingInfra),
         graphs.startGraphite.bind(null, existingInfra),
         sftp.startSftp.bind(null, existingInfra),
         addons.startServices.bind(null, existingInfra),
@@ -73,7 +74,7 @@ function start(callback) {
     });
 }
 
-function stopAllTasks(callback) {
+function stop(callback) {
     tasks.stopAllTasks(callback);
 }
 
@@ -129,37 +130,42 @@ function pruneInfraImages(callback) {
     }, callback);
 }
 
-function removeAllContainers(existingInfra, callback) {
-    debug('removeAllContainers: removing all containers for infra upgrade');
+function stopContainers(existingInfra, callback) {
+    // always stop addons to restart them on any infra change, regardless of minor or major update
+    if (existingInfra.version !== infra.version) {
+        // TODO: only nuke containers with isCloudronManaged=true
+        debug('stopping all containers for infra upgrade');
+        async.series([
+            shell.exec.bind(null, 'stopContainers', 'docker ps -qa --filter \'network=cloudron\' | xargs --no-run-if-empty docker stop'),
+            shell.exec.bind(null, 'stopContainers', 'docker ps -qa --filter \'network=cloudron\' | xargs --no-run-if-empty docker rm -f')
+        ], callback);
+    } else {
+        assert(typeof infra.images, 'object');
+        var changedAddons = [ ];
+        for (var imageName in existingInfra.images) { // do not use infra.images because we can only stop things which are existing
+            if (infra.images[imageName].tag !== existingInfra.images[imageName].tag) changedAddons.push(imageName);
+        }
 
-    async.series([
-        shell.exec.bind(null, 'removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker stop'),
-        shell.exec.bind(null, 'removeAllContainers', 'docker ps -qa --filter \'label=isCloudronManaged\' | xargs --no-run-if-empty docker rm -f')
-    ], callback);
+        debug('stopContainer: stopping addons for incremental infra update: %j', changedAddons);
+        let filterArg = changedAddons.map(function (c) { return `--filter 'name=${c}'`; }).join(' '); // name=c matches *c*. required for redis-{appid}
+        // ignore error if container not found (and fail later) so that this code works across restarts
+        async.series([
+            shell.exec.bind(null, 'stopContainers', `docker ps -qa ${filterArg} --filter 'network=cloudron' | xargs --no-run-if-empty docker stop || true`),
+            shell.exec.bind(null, 'stopContainers', `docker ps -qa ${filterArg} --filter 'network=cloudron' | xargs --no-run-if-empty docker rm -f || true`)
+        ], callback);
+    }
 }
 
-function markApps(existingInfra, callback) {
+function startApps(existingInfra, callback) {
     if (existingInfra.version === 'none') { // cloudron is being restored from backup
-        debug('markApps: restoring installed apps');
+        debug('startApps: restoring installed apps');
         apps.restoreInstalledApps(callback);
     } else if (existingInfra.version !== infra.version) {
-        debug('markApps: reconfiguring installed apps');
+        debug('startApps: reconfiguring installed apps');
         reverseProxy.removeAppConfigs(); // should we change the cert location, nginx will not start
         apps.configureInstalledApps(callback);
     } else {
-        let changedAddons = [];
-        if (infra.images.mysql.tag !== existingInfra.images.mysql.tag) changedAddons.push('mysql');
-        if (infra.images.postgresql.tag !== existingInfra.images.postgresql.tag) changedAddons.push('postgresql');
-        if (infra.images.mongodb.tag !== existingInfra.images.mongodb.tag) changedAddons.push('mongodb');
-        if (infra.images.redis.tag !== existingInfra.images.redis.tag) changedAddons.push('redis');
-
-        if (changedAddons.length) {
-            // restart apps if docker image changes since the IP changes and any "persistent" connections fail
-            debug(`markApps: changedAddons: ${JSON.stringify(changedAddons)}`);
-            apps.restartAppsUsingAddons(changedAddons, callback);
-        } else {
-            debug('markApps: apps are already uptodate');
-            callback();
-        }
+        debug('startApps: apps are already uptodate');
+        callback();
     }
 }

src/provision.js

@@ -4,10 +4,13 @@ exports = module.exports = {
     setup: setup,
     restore: restore,
     activate: activate,
-    getStatus: getStatus
+    getStatus: getStatus,
+
+    autoRegister: autoRegister
 };
 
-var assert = require('assert'),
+var appstore = require('./appstore.js'),
+    assert = require('assert'),
     async = require('async'),
     backups = require('./backups.js'),
     BoxError = require('./boxerror.js'),
@@ -16,7 +19,10 @@ var assert = require('assert'),
     debug = require('debug')('box:provision'),
     domains = require('./domains.js'),
     eventlog = require('./eventlog.js'),
+    fs = require('fs'),
     mail = require('./mail.js'),
+    paths = require('./paths.js'),
+    safe = require('safetydance'),
     semver = require('semver'),
     settings = require('./settings.js'),
     sysinfo = require('./sysinfo.js'),
@@ -47,6 +53,27 @@ function setProgress(task, message, callback) {
     callback();
 }
 
+function autoRegister(domain, callback) {
+    assert.strictEqual(typeof domain, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    if (!fs.existsSync(paths.LICENSE_FILE)) return callback();
+
+    const license = safe.fs.readFileSync(paths.LICENSE_FILE, 'utf8');
+    if (!license) return callback(new BoxError(BoxError.LICENSE_ERROR, 'Cannot read license'));
+
+    debug('Auto-registering cloudron');
+
+    appstore.registerWithLicense(license.trim(), domain, function (error) {
+        if (error && error.reason !== BoxError.CONFLICT) { // not already registered
+            debug('Failed to auto-register cloudron', error);
+            return callback(new BoxError(BoxError.LICENSE_ERROR, 'Failed to auto-register Cloudron with license. Please contact support@cloudron.io'));
+        }
+
+        callback();
+    });
+}
+
 function unprovision(callback) {
     assert.strictEqual(typeof callback, 'function');
 
@@ -94,8 +121,7 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
                 provider: dnsConfig.provider,
                 config: dnsConfig.config,
                 fallbackCertificate: dnsConfig.fallbackCertificate || null,
-                tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' },
-                dkimSelector: 'cloudron'
+                tlsConfig: dnsConfig.tlsConfig || { provider: 'letsencrypt-prod' }
             };
 
             domains.add(domain, data, auditSource, function (error) {
@@ -107,9 +133,11 @@ function setup(dnsConfig, sysinfoConfig, auditSource, callback) {
                     callback(); // now that args are validated run the task in the background
 
                     async.series([
+                        autoRegister.bind(null, domain),
                         settings.setSysinfoConfig.bind(null, sysinfoConfig),
                         domains.prepareDashboardDomain.bind(null, domain, auditSource, (progress) => setProgress('setup', progress.message, NOOP_CALLBACK)),
                         cloudron.setDashboardDomain.bind(null, domain, auditSource),
+                        mail.addDomain.bind(null, domain), // this relies on settings.mailFqdn() and settings.adminDomain()
                         setProgress.bind(null, 'setup', 'Done'),
                         eventlog.add.bind(null, eventlog.ACTION_PROVISION, auditSource, { })
                     ], function (error) {
@@ -178,16 +206,9 @@ function restore(backupConfig, backupId, version, sysinfoConfig, auditSource, ca
         if (error) return done(error);
         if (activated) return done(new BoxError(BoxError.CONFLICT, 'Already activated. Restore with a fresh Cloudron installation.'));
 
-        backups.testProviderConfig(backupConfig, function (error) {
+        backups.testConfig(backupConfig, function (error) {
             if (error) return done(error);
 
-            if ('password' in backupConfig) {
-                backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
-                delete backupConfig.password;
-            } else {
-                backupConfig.encryption = null;
-            }
-
             sysinfo.testConfig(sysinfoConfig, function (error) {
                 if (error) return done(error);
 
@@ -226,11 +247,11 @@ function getStatus(callback) {
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
                 webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
+                provider: settings.provider(),
                 cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
                 footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,
                 activated: activated,
-                provider: settings.provider() // used by setup wizard of marketplace images
             }, gProvisionStatus));
         });
     });

src/reverseproxy.js

@@ -27,7 +27,7 @@ exports = module.exports = {
     removeAppConfigs: removeAppConfigs,
 
     // exported for testing
-    _getAcmeApi: getAcmeApi
+    _getCertApi: getCertApi
 };
 
 var acme2 = require('./cert/acme2.js'),
@@ -35,12 +35,14 @@ var acme2 = require('./cert/acme2.js'),
     assert = require('assert'),
     async = require('async'),
     BoxError = require('./boxerror.js'),
+    caas = require('./cert/caas.js'),
     constants = require('./constants.js'),
     crypto = require('crypto'),
     debug = require('debug')('box:reverseproxy'),
     domains = require('./domains.js'),
     ejs = require('ejs'),
     eventlog = require('./eventlog.js'),
+    fallback = require('./cert/fallback.js'),
     fs = require('fs'),
     mail = require('./mail.js'),
     os = require('os'),
@@ -57,23 +59,27 @@ var acme2 = require('./cert/acme2.js'),
 var NGINX_APPCONFIG_EJS = fs.readFileSync(__dirname + '/nginxconfig.ejs', { encoding: 'utf8' }),
     RELOAD_NGINX_CMD = path.join(__dirname, 'scripts/reloadnginx.sh');
 
-function getAcmeApi(domainObject, callback) {
+function getCertApi(domainObject, callback) {
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    const api = acme2;
+    if (domainObject.tlsConfig.provider === 'fallback') return callback(null, fallback, { fallback: true });
 
-    let options = { prod: false, performHttpAuthorization: false, wildcard: false, email: '' };
-    options.prod = domainObject.tlsConfig.provider.match(/.*-prod/) !== null; // matches 'le-prod' or 'letsencrypt-prod'
-    options.performHttpAuthorization = domainObject.provider.match(/noop|manual|wildcard/) !== null;
-    options.wildcard = !!domainObject.tlsConfig.wildcard;
+    var api = domainObject.tlsConfig.provider === 'caas' ? caas : acme2;
+
+    var options = { prod: false, performHttpAuthorization: false, wildcard: false, email: '' };
+    if (domainObject.tlsConfig.provider !== 'caas') {
+        options.prod = domainObject.tlsConfig.provider.match(/.*-prod/) !== null; // matches 'le-prod' or 'letsencrypt-prod'
+        options.performHttpAuthorization = domainObject.provider.match(/noop|manual|wildcard/) !== null;
+        options.wildcard = !!domainObject.tlsConfig.wildcard;
+    }
 
     // registering user with an email requires A or MX record (https://github.com/letsencrypt/boulder/issues/1197)
     // we cannot use admin@fqdn because the user might not have set it up.
     // we simply update the account with the latest email we have each time when getting letsencrypt certs
     // https://github.com/ietf-wg-acme/acme/issues/30
     users.getOwner(function (error, owner) {
-        options.email = error ? 'webmaster@cloudron.io' : owner.email; // can error if not activated yet
+        options.email = error ? 'support@cloudron.io' : owner.email; // can error if not activated yet
 
         callback(null, api, options);
     });
@@ -102,6 +108,8 @@ function providerMatchesSync(domainObject, certFilePath, apiOptions) {
 
     if (!fs.existsSync(certFilePath)) return false; // not found
 
+    if (apiOptions.fallback) return certFilePath.includes('.host.cert');
+
     const subjectAndIssuer = safe.child_process.execSync(`/usr/bin/openssl x509 -noout -subject -issuer -in "${certFilePath}"`, { encoding: 'utf8' });
     if (!subjectAndIssuer) return false; // something bad happenned
 
@@ -138,19 +146,19 @@ function validateCertificate(location, domainObject, certificate) {
     // -checkhost checks for SAN or CN exclusively. SAN takes precedence and if present, ignores the CN.
     const fqdn = domains.fqdn(location, domainObject);
 
-    let result = safe.child_process.execSync(`openssl x509 -noout -checkhost "${fqdn}"`, { encoding: 'utf8', input: cert });
+    var result = safe.child_process.execSync(`openssl x509 -noout -checkhost "${fqdn}"`, { encoding: 'utf8', input: cert });
     if (result === null) return new BoxError(BoxError.BAD_FIELD, 'Unable to get certificate subject:' + safe.error.message, { field: 'cert' });
 
     if (result.indexOf('does match certificate') === -1) return new BoxError(BoxError.BAD_FIELD, `Certificate is not valid for this domain. Expecting ${fqdn}`, { field: 'cert' });
 
-    // check if public key in the cert and private key matches. pkey below works for RSA and ECDSA keys
-    const pubKeyFromCert = safe.child_process.execSync('openssl x509 -noout -pubkey', { encoding: 'utf8', input: cert });
-    if (pubKeyFromCert === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from cert: ${safe.error.message}`, { field: 'cert' });
+    // http://httpd.apache.org/docs/2.0/ssl/ssl_faq.html#verify
+    var certModulus = safe.child_process.execSync('openssl x509 -noout -modulus', { encoding: 'utf8', input: cert });
+    if (certModulus === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get cert modulus: ${safe.error.message}`, { field: 'cert' });
 
-    const pubKeyFromKey = safe.child_process.execSync('openssl pkey -pubout', { encoding: 'utf8', input: key });
-    if (pubKeyFromKey === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get public key from private key: ${safe.error.message}`, { field: 'cert' });
+    var keyModulus = safe.child_process.execSync('openssl rsa -noout -modulus', { encoding: 'utf8', input: key });
+    if (keyModulus === null) return new BoxError(BoxError.BAD_FIELD, `Unable to get key modulus: ${safe.error.message}`, { field: 'cert' });
 
-    if (pubKeyFromCert !== pubKeyFromKey) return new BoxError(BoxError.BAD_FIELD, 'Public key does not match the certificate.', { field: 'cert' });
+    if (certModulus !== keyModulus) return new BoxError(BoxError.BAD_FIELD, 'Key does not match the certificate.', { field: 'cert' });
 
     // check expiration
     result = safe.child_process.execSync('openssl x509 -checkend 0', { encoding: 'utf8', input: cert });
@@ -207,9 +215,15 @@ function setFallbackCertificate(domain, fallback, callback) {
     assert.strictEqual(typeof fallback, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`setFallbackCertificate: setting certs for domain ${domain}`);
-    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${domain}.host.cert`), fallback.cert)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
-    if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${domain}.host.key`), fallback.key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    if (fallback.restricted) { // restricted certs are not backed up
+        debug(`setFallbackCertificate: setting restricted certs for domain ${domain}`);
+        if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`), fallback.cert)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+        if (!safe.fs.writeFileSync(path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`), fallback.key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    } else {
+        debug(`setFallbackCertificate: setting certs for domain ${domain}`);
+        if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${domain}.host.cert`), fallback.cert)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+        if (!safe.fs.writeFileSync(path.join(paths.APP_CERTS_DIR, `${domain}.host.key`), fallback.key)) return callback(new BoxError(BoxError.FS_ERROR, safe.error.message));
+    }
 
     // TODO: maybe the cert is being used by the mail container
     reload(function (error) {
@@ -223,8 +237,15 @@ function getFallbackCertificate(domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    const certFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.cert`);
-    const keyFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.key`);
+    // check for any pre-provisioned (caas) certs. they get first priority
+    var certFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.cert`);
+    var keyFilePath = path.join(paths.NGINX_CERT_DIR, `${domain}.host.key`);
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
+
+    // check for auto-generated or user set fallback certs
+    certFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.cert`);
+    keyFilePath = path.join(paths.APP_CERTS_DIR, `${domain}.host.key`);
 
     callback(null, { certFilePath, keyFilePath });
 }
@@ -246,12 +267,15 @@ function setAppCertificateSync(location, domainObject, certificate) {
     return null;
 }
 
-function getAcmeCertificate(hostname, domainObject, callback) {
+function getCertificateByHostname(hostname, domainObject, callback) {
     assert.strictEqual(typeof hostname, 'string');
     assert.strictEqual(typeof domainObject, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    let certFilePath, keyFilePath;
+    let certFilePath = path.join(paths.APP_CERTS_DIR, `${hostname}.user.cert`);
+    let keyFilePath = path.join(paths.APP_CERTS_DIR, `${hostname}.user.key`);
+
+    if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
 
     if (hostname !== domainObject.domain && domainObject.tlsConfig.wildcard) { // bare domain is not part of wildcard SAN
         let certName = domains.makeWildcard(hostname).replace('*.', '_.');
@@ -274,22 +298,10 @@ function getCertificate(fqdn, domain, callback) {
     assert.strictEqual(typeof domain, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    // 1. user cert always wins
-    // 2. if using fallback provider, return that cert
-    // 3. look for LE certs
-
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
-        // user cert always wins
-        let certFilePath = path.join(paths.APP_CERTS_DIR, `${fqdn}.user.cert`);
-        let keyFilePath = path.join(paths.APP_CERTS_DIR, `${fqdn}.user.key`);
-
-        if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) return callback(null, { certFilePath, keyFilePath });
-
-        if (domainObject.tlsConfig.provider === 'fallback') return getFallbackCertificate(domain, callback);
-
-        getAcmeCertificate(fqdn, domainObject, function (error, result) {
+        getCertificateByHostname(fqdn, domainObject, function (error, result) {
             if (error || result) return callback(error, result);
 
             return getFallbackCertificate(domain, callback);
@@ -317,32 +329,14 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
     domains.get(domain, function (error, domainObject) {
         if (error) return callback(error);
 
-        // user cert always wins
-        let certFilePath = path.join(paths.APP_CERTS_DIR, `${vhost}.user.cert`);
-        let keyFilePath = path.join(paths.APP_CERTS_DIR, `${vhost}.user.key`);
-
-        if (fs.existsSync(certFilePath) && fs.existsSync(keyFilePath)) {
-            debug(`ensureCertificate: ${vhost} will use custom app certs`);
-            return callback(null, { certFilePath, keyFilePath }, { renewed: false });
-        }
-
-        if (domainObject.tlsConfig.provider === 'fallback') {
-            debug(`ensureCertificate: ${vhost} will use fallback certs`);
-
-            return getFallbackCertificate(domain, function (error, bundle) {
-                if (error) return callback(error);
-
-                callback(null, bundle, { renewed: false });
-            });
-        }
-
-        getAcmeApi(domainObject, function (error, acmeApi, apiOptions) {
+        getCertApi(domainObject, function (error, api, apiOptions) {
             if (error) return callback(error);
 
-            getAcmeCertificate(vhost, domainObject, function (_error, currentBundle) {
+            getCertificateByHostname(vhost, domainObject, function (_error, currentBundle) {
                 if (currentBundle) {
                     debug(`ensureCertificate: ${vhost} certificate already exists at ${currentBundle.keyFilePath}`);
 
+                    if (currentBundle.certFilePath.endsWith('.user.cert')) return callback(null, currentBundle, { renewed: false }); // user certs cannot be renewed
                     if (!isExpiringSync(currentBundle.certFilePath, 24 * 30) && providerMatchesSync(domainObject, currentBundle.certFilePath, apiOptions)) return callback(null, currentBundle, { renewed: false });
                     debug(`ensureCertificate: ${vhost} cert require renewal`);
                 } else {
@@ -351,7 +345,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
 
                 debug('ensureCertificate: getting certificate for %s with options %j', vhost, apiOptions);
 
-                acmeApi.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
+                api.getCertificate(vhost, domain, apiOptions, function (error, certFilePath, keyFilePath) {
                     debug(`ensureCertificate: error: ${error ? error.message : 'null'} cert: ${certFilePath || 'null'}`);
 
                     eventlog.add(currentBundle ? eventlog.ACTION_CERTIFICATE_RENEWAL : eventlog.ACTION_CERTIFICATE_NEW, auditSource, { domain: vhost, errorMessage: error ? error.message : '' });
@@ -368,6 +362,7 @@ function ensureCertificate(vhost, domain, auditSource, callback) {
 
                         debug(`ensureCertificate: renewal of ${vhost} failed. using fallback certificates for ${domain}`);
 
+                        // if no cert was returned use fallback. the fallback/caas provider will not provide any for example
                         getFallbackCertificate(domain, function (error, bundle) {
                             if (error) return callback(error);
 

src/routes/apps.js

@@ -4,16 +4,16 @@ exports = module.exports = {
     getApp: getApp,
     getApps: getApps,
     getAppIcon: getAppIcon,
-    install: install,
-    uninstall: uninstall,
-    restore: restore,
+    installApp: installApp,
+    uninstallApp: uninstallApp,
+    restoreApp: restoreApp,
     importApp: importApp,
-    backup: backup,
-    update: update,
+    backupApp: backupApp,
+    updateApp: updateApp,
     getLogs: getLogs,
     getLogStream: getLogStream,
     listBackups: listBackups,
-    repair: repair,
+    repairApp: repairApp,
 
     setAccessRestriction: setAccessRestriction,
     setLabel: setLabel,
@@ -30,20 +30,17 @@ exports = module.exports = {
     setMailbox: setMailbox,
     setLocation: setLocation,
     setDataDir: setDataDir,
-    setBinds: setBinds,
 
-    stop: stop,
-    start: start,
-    restart: restart,
+    stopApp: stopApp,
+    startApp: startApp,
+    restartApp: restartApp,
     exec: exec,
     execWebSocket: execWebSocket,
 
-    clone: clone,
+    cloneApp: cloneApp,
 
     uploadFile: uploadFile,
-    downloadFile: downloadFile,
-
-    load: load
+    downloadFile: downloadFile
 };
 
 var apps = require('../apps.js'),
@@ -54,28 +51,19 @@ var apps = require('../apps.js'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
     safe = require('safetydance'),
-    users = require('../users.js'),
     util = require('util'),
     WebSocket = require('ws');
 
-function load(req, res, next) {
+function getApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.get(req.params.id, function (error, result) {
+    apps.get(req.params.id, function (error, app) {
         if (error) return next(BoxError.toHttpError(error));
 
-        req.resource = result;
-
-        next();
+        next(new HttpSuccess(200, apps.removeInternalFields(app)));
     });
 }
 
-function getApp(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
-
-    next(new HttpSuccess(200, apps.removeInternalFields(req.resource)));
-}
-
 function getApps(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
@@ -89,19 +77,19 @@ function getApps(req, res, next) {
 }
 
 function getAppIcon(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.getIconPath(req.resource, { original: req.query.original }, function (error, iconPath) {
+    apps.getIconPath(req.params.id, { original: req.query.original }, function (error, iconPath) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.sendFile(iconPath);
     });
 }
 
-function install(req, res, next) {
+function installApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
-    const data = req.body;
+    var data = req.body;
 
     // atleast one
     if ('manifest' in data && typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
@@ -145,28 +133,20 @@ function install(req, res, next) {
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
-    apps.downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
+    apps.install(data, req.user, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
-        if (safe.query(manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to install app with docker addon'));
-
-        data.appStoreId = appStoreId;
-        data.manifest = manifest;
-        apps.install(data, auditSource.fromRequest(req), function (error, result) {
-            if (error) return next(BoxError.toHttpError(error));
-
-            next(new HttpSuccess(202, { id: result.id, taskId: result.taskId }));
-        });
+        next(new HttpSuccess(202, { id: result.id, taskId: result.taskId }));
     });
 }
 
 function setAccessRestriction(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.accessRestriction !== 'object') return next(new HttpError(400, 'accessRestriction must be an object'));
 
-    apps.setAccessRestriction(req.resource, req.body.accessRestriction, auditSource.fromRequest(req), function (error) {
+    apps.setAccessRestriction(req.params.id, req.body.accessRestriction, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -175,11 +155,11 @@ function setAccessRestriction(req, res, next) {
 
 function setLabel(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.label !== 'string') return next(new HttpError(400, 'label must be a string'));
 
-    apps.setLabel(req.resource, req.body.label, auditSource.fromRequest(req), function (error) {
+    apps.setLabel(req.params.id, req.body.label, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -188,12 +168,12 @@ function setLabel(req, res, next) {
 
 function setTags(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (!Array.isArray(req.body.tags)) return next(new HttpError(400, 'tags must be an array'));
     if (req.body.tags.some((t) => typeof t !== 'string')) return next(new HttpError(400, 'tags array must contain strings'));
 
-    apps.setTags(req.resource, req.body.tags, auditSource.fromRequest(req), function (error) {
+    apps.setTags(req.params.id, req.body.tags, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -202,11 +182,11 @@ function setTags(req, res, next) {
 
 function setIcon(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (req.body.icon !== null && typeof req.body.icon !== 'string') return next(new HttpError(400, 'icon is null or a base-64 image string'));
 
-    apps.setIcon(req.resource, req.body.icon, auditSource.fromRequest(req), function (error) {
+    apps.setIcon(req.params.id, req.body.icon, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -215,11 +195,11 @@ function setIcon(req, res, next) {
 
 function setMemoryLimit(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.memoryLimit !== 'number') return next(new HttpError(400, 'memoryLimit is not a number'));
 
-    apps.setMemoryLimit(req.resource, req.body.memoryLimit, auditSource.fromRequest(req), function (error, result) {
+    apps.setMemoryLimit(req.params.id, req.body.memoryLimit, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -228,11 +208,11 @@ function setMemoryLimit(req, res, next) {
 
 function setCpuShares(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.cpuShares !== 'number') return next(new HttpError(400, 'cpuShares is not a number'));
 
-    apps.setCpuShares(req.resource, req.body.cpuShares, auditSource.fromRequest(req), function (error, result) {
+    apps.setCpuShares(req.params.id, req.body.cpuShares, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -241,11 +221,11 @@ function setCpuShares(req, res, next) {
 
 function setAutomaticBackup(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enable must be a boolean'));
 
-    apps.setAutomaticBackup(req.resource, req.body.enable, auditSource.fromRequest(req), function (error) {
+    apps.setAutomaticBackup(req.params.id, req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -254,11 +234,11 @@ function setAutomaticBackup(req, res, next) {
 
 function setAutomaticUpdate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.enable !== 'boolean') return next(new HttpError(400, 'enable must be a boolean'));
 
-    apps.setAutomaticUpdate(req.resource, req.body.enable, auditSource.fromRequest(req), function (error) {
+    apps.setAutomaticUpdate(req.params.id, req.body.enable, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -267,13 +247,13 @@ function setAutomaticUpdate(req, res, next) {
 
 function setReverseProxyConfig(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (req.body.robotsTxt !== null && typeof req.body.robotsTxt !== 'string') return next(new HttpError(400, 'robotsTxt is not a string'));
 
     if (req.body.csp !== null && typeof req.body.csp !== 'string') return next(new HttpError(400, 'csp is not a string'));
 
-    apps.setReverseProxyConfig(req.resource, req.body, auditSource.fromRequest(req), function (error) {
+    apps.setReverseProxyConfig(req.params.id, req.body, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -282,14 +262,14 @@ function setReverseProxyConfig(req, res, next) {
 
 function setCertificate(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (req.body.key !== null && typeof req.body.cert !== 'string') return next(new HttpError(400, 'cert must be a string'));
     if (req.body.cert !== null && typeof req.body.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (req.body.cert && !req.body.key) return next(new HttpError(400, 'key must be provided'));
     if (!req.body.cert && req.body.key) return next(new HttpError(400, 'cert must be provided'));
 
-    apps.setCertificate(req.resource, req.body, auditSource.fromRequest(req), function (error) {
+    apps.setCertificate(req.params.id, req.body, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, {}));
@@ -298,12 +278,12 @@ function setCertificate(req, res, next) {
 
 function setEnvironment(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (!req.body.env || typeof req.body.env !== 'object') return next(new HttpError(400, 'env must be an object'));
     if (Object.keys(req.body.env).some((key) => typeof req.body.env[key] !== 'string')) return next(new HttpError(400, 'env must contain values as strings'));
 
-    apps.setEnvironment(req.resource, req.body.env, auditSource.fromRequest(req), function (error, result) {
+    apps.setEnvironment(req.params.id, req.body.env, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -312,11 +292,11 @@ function setEnvironment(req, res, next) {
 
 function setDebugMode(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (req.body.debugMode !== null && typeof req.body.debugMode !== 'object') return next(new HttpError(400, 'debugMode must be an object'));
 
-    apps.setDebugMode(req.resource, req.body.debugMode, auditSource.fromRequest(req), function (error, result) {
+    apps.setDebugMode(req.params.id, req.body.debugMode, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -325,12 +305,12 @@ function setDebugMode(req, res, next) {
 
 function setMailbox(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (req.body.mailboxName !== null && typeof req.body.mailboxName !== 'string') return next(new HttpError(400, 'mailboxName must be a string'));
     if (typeof req.body.mailboxDomain !== 'string') return next(new HttpError(400, 'mailboxDomain must be a string'));
 
-    apps.setMailbox(req.resource, req.body.mailboxName, req.body.mailboxDomain, auditSource.fromRequest(req), function (error, result) {
+    apps.setMailbox(req.params.id, req.body.mailboxName, req.body.mailboxDomain, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -339,7 +319,7 @@ function setMailbox(req, res, next) {
 
 function setLocation(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.body.location !== 'string') return next(new HttpError(400, 'location must be string')); // location may be an empty string
     if (!req.body.domain) return next(new HttpError(400, 'domain is required'));
@@ -354,7 +334,7 @@ function setLocation(req, res, next) {
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
-    apps.setLocation(req.resource, req.body, auditSource.fromRequest(req), function (error, result) {
+    apps.setLocation(req.params.id, req.body, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -363,49 +343,47 @@ function setLocation(req, res, next) {
 
 function setDataDir(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (req.body.dataDir !== null && typeof req.body.dataDir !== 'string') return next(new HttpError(400, 'dataDir must be a string'));
 
-    apps.setDataDir(req.resource, req.body.dataDir, auditSource.fromRequest(req), function (error, result) {
+    apps.setDataDir(req.params.id, req.body.dataDir, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function repair(req, res, next) {
+function repairApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     const data = req.body;
 
     if ('manifest' in data) {
         if (!data.manifest || typeof data.manifest !== 'object') return next(new HttpError(400, 'manifest must be an object'));
-
-        if (safe.query(data.manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to repair app with docker addon'));
     }
 
     if ('dockerImage' in data) {
         if (!data.dockerImage || typeof data.dockerImage !== 'string') return next(new HttpError(400, 'dockerImage must be a string'));
     }
 
-    apps.repair(req.resource, data, auditSource.fromRequest(req), function (error, result) {
+    apps.repair(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function restore(req, res, next) {
+function restoreApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
     if (!data.backupId || typeof data.backupId !== 'string') return next(new HttpError(400, 'backupId must be non-empty string'));
 
-    apps.restore(req.resource, data.backupId, auditSource.fromRequest(req), function (error, result) {
+    apps.restore(req.params.id, data.backupId, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
@@ -414,7 +392,7 @@ function restore(req, res, next) {
 
 function importApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
@@ -428,7 +406,7 @@ function importApp(req, res, next) {
 
         if (req.body.backupConfig) {
             if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
-            if ('password' in backupConfig && typeof backupConfig.password !== 'string') return next(new HttpError(400, 'password must be a string'));
+            if ('key' in backupConfig && typeof backupConfig.key !== 'string') return next(new HttpError(400, 'key must be a string'));
             if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
 
             // testing backup config can take sometime
@@ -436,16 +414,16 @@ function importApp(req, res, next) {
         }
     }
 
-    apps.importApp(req.resource, data, auditSource.fromRequest(req), function (error, result) {
+    apps.importApp(req.params.id, data, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function clone(req, res, next) {
+function cloneApp(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var data = req.body;
 
@@ -456,66 +434,66 @@ function clone(req, res, next) {
 
     if ('overwriteDns' in req.body && typeof req.body.overwriteDns !== 'boolean') return next(new HttpError(400, 'overwriteDns must be boolean'));
 
-    apps.clone(req.resource, data, req.user, auditSource.fromRequest(req), function (error, result) {
+    apps.clone(req.params.id, data, req.user, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(201, { id: result.id, taskId: result.taskId }));
     });
 }
 
-function backup(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+function backupApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.backup(req.resource, function (error, result) {
+    apps.backup(req.params.id, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function uninstall(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+function uninstallApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.uninstall(req.resource, auditSource.fromRequest(req), function (error, result) {
+    apps.uninstall(req.params.id, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function start(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+function startApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.start(req.resource, auditSource.fromRequest(req), function (error, result) {
+    apps.start(req.params.id, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function stop(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+function stopApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.stop(req.resource, auditSource.fromRequest(req), function (error, result) {
+    apps.stop(req.params.id, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function restart(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+function restartApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
 
-    apps.restart(req.resource, auditSource.fromRequest(req), function (error, result) {
+    apps.restart(req.params.id, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
-function update(req, res, next) {
+function updateApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
     assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
 
     var data = req.body;
 
@@ -527,24 +505,16 @@ function update(req, res, next) {
     if ('skipBackup' in data && typeof data.skipBackup !== 'boolean') return next(new HttpError(400, 'skipBackup must be a boolean'));
     if ('force' in data && typeof data.force !== 'boolean') return next(new HttpError(400, 'force must be a boolean'));
 
-    apps.downloadManifest(data.appStoreId, data.manifest, function (error, appStoreId, manifest) {
+    apps.update(req.params.id, req.body, auditSource.fromRequest(req), function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
-        if (safe.query(manifest, 'addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is required to update app with docker addon'));
-
-        data.appStoreId = appStoreId;
-        data.manifest = manifest;
-        apps.update(req.resource, data, auditSource.fromRequest(req), function (error, result) {
-            if (error) return next(BoxError.toHttpError(error));
-
-            next(new HttpSuccess(202, { taskId: result.taskId }));
-        });
+        next(new HttpSuccess(202, { taskId: result.taskId }));
     });
 }
 
 // this route is for streaming logs
 function getLogStream(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10; // we ignore last-event-id
     if (isNaN(lines)) return next(new HttpError(400, 'lines must be a valid number'));
@@ -559,7 +529,7 @@ function getLogStream(req, res, next) {
         format: 'json'
     };
 
-    apps.getLogs(req.resource, options, function (error, logStream) {
+    apps.getLogs(req.params.id, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -581,7 +551,7 @@ function getLogStream(req, res, next) {
 }
 
 function getLogs(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var lines = 'lines' in req.query ? parseInt(req.query.lines, 10) : 10;
     if (isNaN(lines)) return next(new HttpError(400, 'lines must be a number'));
@@ -592,7 +562,7 @@ function getLogs(req, res, next) {
         format: req.query.format || 'json'
     };
 
-    apps.getLogs(req.resource, options, function (error, logStream) {
+    apps.getLogs(req.params.id, options, function (error, logStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         res.writeHead(200, {
@@ -628,7 +598,7 @@ function demuxStream(stream, stdin) {
 }
 
 function exec(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var cmd = null;
     if (req.query.cmd) {
@@ -642,16 +612,13 @@ function exec(req, res, next) {
     var rows = req.query.rows ? parseInt(req.query.rows, 10) : null;
     if (isNaN(rows)) return next(new HttpError(400, 'rows must be a number'));
 
-    var tty = req.query.tty === 'true';
+    var tty = req.query.tty === 'true' ? true : false;
 
-    if (safe.query(req.resource, 'manifest.addons.docker') && req.user.role !== users.ROLE_OWNER) return next(new HttpError(403, '"owner" role is requied to exec app with docker addon'));
-
-    // in a badly configured reverse proxy, we might be here without an upgrade
-    if (req.headers['upgrade'] !== 'tcp') return next(new HttpError(404, 'exec requires TCP upgrade'));
-
-    apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
+    apps.exec(req.params.id, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
         if (error) return next(BoxError.toHttpError(error));
 
+        if (req.headers['upgrade'] !== 'tcp') return next(new HttpError(404, 'exec requires TCP upgrade'));
+
         req.clearTimeout();
         res.sendUpgradeHandshake();
 
@@ -669,7 +636,7 @@ function exec(req, res, next) {
 }
 
 function execWebSocket(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var cmd = null;
     if (req.query.cmd) {
@@ -685,10 +652,7 @@ function execWebSocket(req, res, next) {
 
     var tty = req.query.tty === 'true' ? true : false;
 
-    // in a badly configured reverse proxy, we might be here without an upgrade
-    if (req.headers['upgrade'] !== 'websocket') return next(new HttpError(404, 'exec requires websocket'));
-
-    apps.exec(req.resource, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
+    apps.exec(req.params.id, { cmd: cmd, rows: rows, columns: columns, tty: tty }, function (error, duplexStream) {
         if (error) return next(BoxError.toHttpError(error));
 
         req.clearTimeout();
@@ -718,7 +682,7 @@ function execWebSocket(req, res, next) {
 }
 
 function listBackups(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
     if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a postive number'));
@@ -726,7 +690,7 @@ function listBackups(req, res, next) {
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
     if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
 
-    apps.listBackups(req.resource, page, perPage, function (error, result) {
+    apps.listBackups(page, perPage, req.params.id, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { backups: result }));
@@ -734,12 +698,12 @@ function listBackups(req, res, next) {
 }
 
 function uploadFile(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.query.file !== 'string' || !req.query.file) return next(new HttpError(400, 'file query argument must be provided'));
     if (!req.files.file) return next(new HttpError(400, 'file must be provided as multipart'));
 
-    apps.uploadFile(req.resource, req.files.file.path, req.query.file, function (error) {
+    apps.uploadFile(req.params.id, req.files.file.path, req.query.file, function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(202, {}));
@@ -747,11 +711,11 @@ function uploadFile(req, res, next) {
 }
 
 function downloadFile(req, res, next) {
-    assert.strictEqual(typeof req.resource, 'object');
+    assert.strictEqual(typeof req.params.id, 'string');
 
     if (typeof req.query.file !== 'string' || !req.query.file) return next(new HttpError(400, 'file query argument must be provided'));
 
-    apps.downloadFile(req.resource, req.query.file, function (error, stream, info) {
+    apps.downloadFile(req.params.id, req.query.file, function (error, stream, info) {
         if (error) return next(BoxError.toHttpError(error));
 
         var headers = {
@@ -765,23 +729,3 @@ function downloadFile(req, res, next) {
         stream.pipe(res);
     });
 }
-
-function setBinds(req, res, next) {
-    assert.strictEqual(typeof req.body, 'object');
-    assert.strictEqual(typeof req.resource, 'object');
-
-    if (!req.body.binds || typeof req.body.binds !== 'object') return next(new HttpError(400, 'binds should be an object'));
-
-    for (let name of Object.keys(req.body.binds)) {
-        if (!req.body.binds[name] || typeof req.body.binds[name] !== 'object') return next(new HttpError(400, 'each bind should be an object'));
-        if (typeof req.body.binds[name].hostPath !== 'string') return next(new HttpError(400, 'hostPath must be a string'));
-        if (typeof req.body.binds[name].readOnly !== 'boolean') return next(new HttpError(400, 'readOnly must be a boolean'));
-    }
-
-    apps.setBinds(req.resource, req.body.binds, auditSource.fromRequest(req), function (error, result) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(202, { taskId: result.taskId }));
-    });
-}
-

src/routes/backups.js

@@ -3,11 +3,11 @@
 exports = module.exports = {
     list: list,
     startBackup: startBackup,
-    cleanup: cleanup,
-    check: check
+    cleanup: cleanup
 };
 
 let auditSource = require('../auditsource.js'),
+    backupdb = require('../backupdb.js'),
     backups = require('../backups.js'),
     BoxError = require('../boxerror.js'),
     HttpError = require('connect-lastmile').HttpError,
@@ -20,7 +20,7 @@ function list(req, res, next) {
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
     if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a postive number'));
 
-    backups.getByIdentifierAndStatePaged(backups.BACKUP_IDENTIFIER_BOX, backups.BACKUP_STATE_NORMAL, page, perPage, function (error, result) {
+    backups.getByStatePaged(backupdb.BACKUP_STATE_NORMAL, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { backups: result }));
@@ -42,11 +42,3 @@ function cleanup(req, res, next) {
         next(new HttpSuccess(202, { taskId }));
     });
 }
-
-function check(req, res, next) {
-    backups.checkConfiguration(function (error, message) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, { ok: !message, message: message }));
-    });
-}

src/routes/cloudron.js

@@ -86,8 +86,8 @@ function logout(req, res) {
 function passwordResetRequest(req, res, next) {
     if (!req.body.identifier || typeof req.body.identifier !== 'string') return next(new HttpError(401, 'A identifier must be non-empty string'));
 
-    users.sendPasswordResetByIdentifier(req.body.identifier, function (error) {
-        if (error && error.reason !== BoxError.NOT_FOUND) return next(BoxError.toHttpError(error));
+    users.resetPasswordByIdentifier(req.body.identifier, function (error) {
+        if (error && error.reason !== BoxError.NOT_FOUND) console.error(error);
 
         next(new HttpSuccess(202, {}));
     });
@@ -102,17 +102,15 @@ function passwordReset(req, res, next) {
     users.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return next(new HttpError(401, 'Invalid resetToken'));
 
-        // if you fix the duration here, the emails and UI have to be fixed as well
-        if (Date.now() - userObject.resetTokenCreationTime > 7 * 24 * 60 * 60 * 1000) return next(new HttpError(401, 'Token expired'));
         if (!userObject.username) return next(new HttpError(409, 'No username set'));
 
         // setPassword clears the resetToken
         users.setPassword(userObject, req.body.password, function (error) {
             if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
-            if (error) return next(BoxError.toHttpError(error));
+            if (error) return next(new HttpError(500, error));
 
             tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
-                if (error) return next(BoxError.toHttpError(error));
+                if (error) return next(new HttpError(500, error));
 
                 next(new HttpSuccess(202, { accessToken: result.accessToken }));
             });
@@ -123,23 +121,35 @@ function passwordReset(req, res, next) {
 function setupAccount(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
 
+    if (!req.body.email || typeof req.body.email !== 'string') return next(new HttpError(400, 'email must be a non-empty string'));
     if (!req.body.resetToken || typeof req.body.resetToken !== 'string') return next(new HttpError(400, 'resetToken must be a non-empty string'));
     if (!req.body.password || typeof req.body.password !== 'string') return next(new HttpError(400, 'password must be a non-empty string'));
-
-    // only sent if profile is not locked
-    if ('username' in req.body && typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a non-empty string'));
-    if ('displayName' in req.body && typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be a non-empty string'));
+    if (!req.body.username || typeof req.body.username !== 'string') return next(new HttpError(400, 'username must be a non-empty string'));
+    if (!req.body.displayName || typeof req.body.displayName !== 'string') return next(new HttpError(400, 'displayName must be a non-empty string'));
 
     users.getByResetToken(req.body.resetToken, function (error, userObject) {
         if (error) return next(new HttpError(401, 'Invalid Reset Token'));
 
-        // if you fix the duration here, the emails and UI have to be fixed as well
-        if (Date.now() - userObject.resetTokenCreationTime > 24 * 60 * 60 * 1000) return next(new HttpError(401, 'Token expired'));
+        users.update(userObject, { username: req.body.username, displayName: req.body.displayName }, auditSource.fromRequest(req), function (error) {
+            if (error && error.reason === BoxError.ALREADY_EXISTS) return next(new HttpError(409, 'Username already used'));
+            if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+            if (error && error.reason === BoxError.NOT_FOUND) return next(new HttpError(404, 'No such user'));
+            if (error) return next(new HttpError(500, error));
 
-        users.setupAccount(userObject, req.body, auditSource.fromRequest(req), function (error, accessToken) {
-            if (error) return next(BoxError.toHttpError(error));
+            userObject.username = req.body.username;
+            userObject.displayName = req.body.displayName;
 
-            next(new HttpSuccess(201, { accessToken }));
+            // setPassword clears the resetToken
+            users.setPassword(userObject, req.body.password, function (error) {
+                if (error && error.reason === BoxError.BAD_FIELD) return next(new HttpError(400, error.message));
+                if (error) return next(new HttpError(500, error));
+
+                tokens.add(tokens.ID_WEBADMIN, userObject.id, Date.now() + constants.DEFAULT_TOKEN_EXPIRATION, {}, function (error, result) {
+                    if (error) return next(new HttpError(500, error));
+
+                    next(new HttpSuccess(201, { accessToken: result.accessToken }));
+                });
+            });
         });
     });
 }
@@ -205,8 +215,8 @@ function checkForUpdates(req, res, next) {
     req.clearTimeout();
 
     async.series([
-        (done) => updateChecker.checkAppUpdates({ automatic: false }, done),
-        (done) => updateChecker.checkBoxUpdates({ automatic: false }, done),
+        updateChecker.checkAppUpdates,
+        updateChecker.checkBoxUpdates
     ], function () {
         next(new HttpSuccess(200, { update: updateChecker.getUpdateInfo() }));
     });

src/routes/domains.js

@@ -33,6 +33,7 @@ function add(req, res, next) {
         let fallbackCertificate = req.body.fallbackCertificate;
         if (!fallbackCertificate.cert || typeof fallbackCertificate.cert !== 'string') return next(new HttpError(400, 'fallbackCertificate.cert must be a string'));
         if (!fallbackCertificate.key || typeof fallbackCertificate.key !== 'string') return next(new HttpError(400, 'fallbackCertificate.key must be a string'));
+        if ('restricted' in fallbackCertificate && typeof fallbackCertificate.restricted !== 'boolean') return next(new HttpError(400, 'fallbackCertificate.restricted must be a boolean'));
     }
 
     if ('tlsConfig' in req.body) {
@@ -94,6 +95,7 @@ function update(req, res, next) {
         let fallbackCertificate = req.body.fallbackCertificate;
         if (!fallbackCertificate.cert || typeof fallbackCertificate.cert !== 'string') return next(new HttpError(400, 'fallbackCertificate.cert must be a string'));
         if (!fallbackCertificate.key || typeof fallbackCertificate.key !== 'string') return next(new HttpError(400, 'fallbackCertificate.key must be a string'));
+        if ('restricted' in fallbackCertificate && typeof fallbackCertificate.restricted !== 'boolean') return next(new HttpError(400, 'fallbackCertificate.restricted must be a boolean'));
     }
 
     if ('tlsConfig' in req.body) {

src/routes/filemanager.js

@@ -1,43 +0,0 @@
-'use strict';
-
-exports = module.exports = {
-    proxy
-};
-
-var assert = require('assert'),
-    BoxError = require('../boxerror.js'),
-    docker = require('../docker.js'),
-    middleware = require('../middleware/index.js'),
-    HttpError = require('connect-lastmile').HttpError,
-    safe = require('safetydance'),
-    url = require('url');
-
-function proxy(req, res, next) {
-    assert.strictEqual(typeof req.params.id, 'string');
-
-    const appId = req.params.id;
-
-    req.clearTimeout();
-
-    docker.inspect('sftp', function (error, result) {
-        if (error)return next(BoxError.toHttpError(error));
-
-        const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
-        if (!ip) return next(new BoxError(BoxError.INACTIVE, 'Error getting IP of sftp service'));
-
-        req.url = req.originalUrl.replace(`/api/v1/apps/${appId}/files`, `/files/${appId}`);
-
-        const proxyOptions = url.parse(`https://${ip}:3000`);
-        proxyOptions.rejectUnauthorized = false;
-        const fileManagerProxy = middleware.proxy(proxyOptions);
-
-        fileManagerProxy(req, res, function (error) {
-            if (!error) return next();
-
-            if (error.code === 'ECONNREFUSED') return next(new HttpError(424, 'Unable to connect to filemanager server'));
-            if (error.code === 'ECONNRESET') return next(new HttpError(424, 'Unable to query filemanager server'));
-
-            next(new HttpError(500, error));
-        });
-    });
-}

src/routes/groups.js

@@ -20,9 +20,7 @@ function create(req, res, next) {
 
     if (typeof req.body.name !== 'string') return next(new HttpError(400, 'name must be string'));
 
-    var source = ''; // means local
-
-    groups.create(req.body.name, source, function (error, group) {
+    groups.create(req.body.name, function (error, group) {
         if (error) return next(BoxError.toHttpError(error));
 
         var groupInfo = {
@@ -71,7 +69,7 @@ function updateMembers(req, res, next) {
 }
 
 function list(req, res, next) {
-    groups.getAllWithMembers(function (error, result) {
+    groups.getAll(function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { groups: result }));

src/routes/index.js

@@ -10,7 +10,6 @@ exports = module.exports = {
     cloudron: require('./cloudron.js'),
     domains: require('./domains.js'),
     eventlog: require('./eventlog.js'),
-    filemanager: require('./filemanager.js'),
     graphs: require('./graphs.js'),
     groups: require('./groups.js'),
     mail: require('./mail.js'),

src/routes/mail.js

@@ -1,35 +1,36 @@
 'use strict';
 
 exports = module.exports = {
-    getDomain,
+    getDomain: getDomain,
+    addDomain: addDomain,
+    removeDomain: removeDomain,
 
-    setDnsRecords,
+    setDnsRecords: setDnsRecords,
 
-    getStatus,
+    getStatus: getStatus,
 
-    setMailFromValidation,
-    setCatchAllAddress,
-    setMailRelay,
-    setMailEnabled,
+    setMailFromValidation: setMailFromValidation,
+    setCatchAllAddress: setCatchAllAddress,
+    setMailRelay: setMailRelay,
+    setMailEnabled: setMailEnabled,
 
-    sendTestMail,
+    sendTestMail: sendTestMail,
 
-    listMailboxes,
-    getMailbox,
-    addMailbox,
-    updateMailbox,
-    removeMailbox,
+    listMailboxes: listMailboxes,
+    getMailbox: getMailbox,
+    addMailbox: addMailbox,
+    updateMailbox: updateMailbox,
+    removeMailbox: removeMailbox,
 
-    getAliases,
-    setAliases,
+    listAliases: listAliases,
+    getAliases: getAliases,
+    setAliases: setAliases,
 
-    getLists,
-    getList,
-    addList,
-    updateList,
-    removeList,
-
-    getMailboxCount
+    getLists: getLists,
+    getList: getList,
+    addList: addList,
+    updateList: updateList,
+    removeList: removeList,
 };
 
 var assert = require('assert'),
@@ -49,6 +50,18 @@ function getDomain(req, res, next) {
     });
 }
 
+function addDomain(req, res, next) {
+    assert.strictEqual(typeof req.body, 'object');
+
+    if (typeof req.body.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
+
+    mail.addDomain(req.body.domain, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(201, { domain: req.body.domain }));
+    });
+}
+
 function setDnsRecords(req, res, next) {
     assert.strictEqual(typeof req.body, 'object');
     assert.strictEqual(typeof req.params.domain, 'string');
@@ -64,6 +77,16 @@ function setDnsRecords(req, res, next) {
     });
 }
 
+function removeDomain(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+
+    mail.removeDomain(req.params.domain, function (error) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(204));
+    });
+}
+
 function getStatus(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
@@ -161,25 +184,13 @@ function listMailboxes(req, res, next) {
     var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
     if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
 
-    if (req.query.search && typeof req.query.search !== 'string') return next(new HttpError(400, 'search must be a string'));
-
-    mail.listMailboxes(req.params.domain, req.query.search || null, page, perPage, function (error, result) {
+    mail.listMailboxes(req.params.domain, page, perPage, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { mailboxes: result }));
     });
 }
 
-function getMailboxCount(req, res, next) {
-    assert.strictEqual(typeof req.params.domain, 'string');
-
-    mail.getMailboxCount(req.params.domain, function (error, count) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        next(new HttpSuccess(200, { count }));
-    });
-}
-
 function getMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
@@ -221,15 +232,29 @@ function removeMailbox(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
 
-    if (typeof req.body.deleteMails !== 'boolean') return next(new HttpError(400, 'deleteMails must be a boolean'));
-
-    mail.removeMailbox(req.params.name, req.params.domain, req.body, auditSource.fromRequest(req), function (error) {
+    mail.removeMailbox(req.params.name, req.params.domain, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(201, {}));
     });
 }
 
+function listAliases(req, res, next) {
+    assert.strictEqual(typeof req.params.domain, 'string');
+
+    var page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
+    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
+
+    var perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
+    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
+
+    mail.listAliases(req.params.domain, page, perPage, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(200, { aliases: result }));
+    });
+}
+
 function getAliases(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
     assert.strictEqual(typeof req.params.name, 'string');
@@ -248,10 +273,8 @@ function setAliases(req, res, next) {
 
     if (!Array.isArray(req.body.aliases)) return next(new HttpError(400, 'aliases must be an array'));
 
-    for (let alias of req.body.aliases) {
-        if (!alias || typeof alias !== 'object') return next(new HttpError(400, 'each alias must have a name and domain'));
-        if (typeof alias.name !== 'string') return next(new HttpError(400, 'name must be a string'));
-        if (typeof alias.domain !== 'string') return next(new HttpError(400, 'domain must be a string'));
+    for (var i = 0; i < req.body.aliases.length; i++) {
+        if (typeof req.body.aliases[i] !== 'string') return next(new HttpError(400, 'alias must be a string'));
     }
 
     mail.setAliases(req.params.name, req.params.domain, req.body.aliases, function (error) {
@@ -264,15 +287,7 @@ function setAliases(req, res, next) {
 function getLists(req, res, next) {
     assert.strictEqual(typeof req.params.domain, 'string');
 
-    const page = typeof req.query.page !== 'undefined' ? parseInt(req.query.page) : 1;
-    if (!page || page < 0) return next(new HttpError(400, 'page query param has to be a positive number'));
-
-    const perPage = typeof req.query.per_page !== 'undefined'? parseInt(req.query.per_page) : 25;
-    if (!perPage || perPage < 0) return next(new HttpError(400, 'per_page query param has to be a positive number'));
-
-    if (req.query.search && typeof req.query.search !== 'string') return next(new HttpError(400, 'search must be a string'));
-
-    mail.getLists(req.params.domain, req.query.search || null, page, perPage, function (error, result) {
+    mail.getLists(req.params.domain, function (error, result) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(200, { lists: result }));
@@ -301,9 +316,8 @@ function addList(req, res, next) {
     for (var i = 0; i < req.body.members.length; i++) {
         if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
     }
-    if (typeof req.body.membersOnly !== 'boolean') return next(new HttpError(400, 'membersOnly must be a boolean'));
 
-    mail.addList(req.body.name, req.params.domain, req.body.members, req.body.membersOnly, auditSource.fromRequest(req), function (error) {
+    mail.addList(req.body.name, req.params.domain, req.body.members, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(201, {}));
@@ -320,9 +334,8 @@ function updateList(req, res, next) {
     for (var i = 0; i < req.body.members.length; i++) {
         if (typeof req.body.members[i] !== 'string') return next(new HttpError(400, 'member must be a string'));
     }
-    if (typeof req.body.membersOnly !== 'boolean') return next(new HttpError(400, 'membersOnly must be a boolean'));
 
-    mail.updateList(req.params.name, req.params.domain, req.body.members, req.body.membersOnly, auditSource.fromRequest(req), function (error) {
+    mail.updateList(req.params.name, req.params.domain, req.body.members, auditSource.fromRequest(req), function (error) {
         if (error) return next(BoxError.toHttpError(error));
 
         next(new HttpSuccess(204));

src/routes/mailserver.js

@@ -21,7 +21,7 @@ function proxy(req, res, next) {
     delete req.headers['authorization'];
     delete req.headers['cookies'];
 
-    addons.getContainerDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
+    addons.getServiceDetails('mail', 'CLOUDRON_MAIL_TOKEN', function (error, addonDetails) {
         if (error) return next(BoxError.toHttpError(error));
 
         parsedUrl.query['access_token'] = addonDetails.token;

src/routes/profile.js

@@ -1,42 +1,35 @@
 'use strict';
 
 exports = module.exports = {
-    authorize,
-    get,
-    update,
-    getAvatar,
-    setAvatar,
-    clearAvatar,
-    changePassword,
-    setTwoFactorAuthenticationSecret,
-    enableTwoFactorAuthentication,
-    disableTwoFactorAuthentication,
+    get: get,
+    update: update,
+    getAvatar: getAvatar,
+    setAvatar: setAvatar,
+    clearAvatar: clearAvatar,
+    changePassword: changePassword,
+    setTwoFactorAuthenticationSecret: setTwoFactorAuthenticationSecret,
+    enableTwoFactorAuthentication: enableTwoFactorAuthentication,
+    disableTwoFactorAuthentication: disableTwoFactorAuthentication
 };
 
 var assert = require('assert'),
     auditSource = require('../auditsource.js'),
     BoxError = require('../boxerror.js'),
+    fs = require('fs'),
     HttpError = require('connect-lastmile').HttpError,
     HttpSuccess = require('connect-lastmile').HttpSuccess,
+    path = require('path'),
+    paths = require('../paths.js'),
+    safe = require('safetydance'),
     users = require('../users.js'),
     settings = require('../settings.js'),
     _ = require('underscore');
 
-function authorize(req, res, next) {
-    assert.strictEqual(typeof req.user, 'object');
-
-    settings.getDirectoryConfig(function (error, directoryConfig) {
-        if (error) return next(BoxError.toHttpError(error));
-
-        if (directoryConfig.lockUserProfiles) return next(new HttpError(403, 'admin has disallowed users from editing profiles'));
-
-        next();
-    });
-}
-
 function get(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
+    const emailHash = require('crypto').createHash('md5').update(req.user.email).digest('hex');
+
     next(new HttpSuccess(200, {
         id: req.user.id,
         username: req.user.username,
@@ -46,7 +39,7 @@ function get(req, res, next) {
         twoFactorAuthenticationEnabled: req.user.twoFactorAuthenticationEnabled,
         role: req.user.role,
         source: req.user.source,
-        avatarUrl: users.getAvatarUrlSync(req.user)
+        avatarUrl: fs.existsSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id)) ? `${settings.adminOrigin()}/api/v1/profile/avatar/${req.user.id}` : `https://www.gravatar.com/avatar/${emailHash}.jpg`
     }));
 }
 
@@ -72,27 +65,21 @@ function setAvatar(req, res, next) {
 
     if (!req.files.avatar) return next(new HttpError(400, 'avatar is missing'));
 
-    users.setAvatar(req.user.id, req.files.avatar.path, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
+    if (!safe.fs.renameSync(req.files.avatar.path, path.join(paths.PROFILE_ICONS_DIR, req.user.id))) return next(new HttpError(500, safe.error));
 
-        next(new HttpSuccess(202, {}));
-    });
+    next(new HttpSuccess(202, {}));
 }
 
 function clearAvatar(req, res, next) {
     assert.strictEqual(typeof req.user, 'object');
 
-    users.clearAvatar(req.user.id, function (error) {
-        if (error) return next(BoxError.toHttpError(error));
+    safe.fs.unlinkSync(path.join(paths.PROFILE_ICONS_DIR, req.user.id));
 
-        next(new HttpSuccess(202, {}));
-    });
+    next(new HttpSuccess(202, {}));
 }
 
 function getAvatar(req, res) {
-    assert.strictEqual(typeof req.params.identifier, 'string');
-
-    res.sendFile(users.getAvatarFileSync(req.params.identifier));
+    res.sendFile(path.join(paths.PROFILE_ICONS_DIR, req.params.identifier));
 }
 
 function changePassword(req, res, next) {

src/routes/provision.js

@@ -98,11 +98,11 @@ function restore(req, res, next) {
 
     var backupConfig = req.body.backupConfig;
     if (typeof backupConfig.provider !== 'string') return next(new HttpError(400, 'provider is required'));
-    if ('password' in backupConfig && typeof backupConfig.password !== 'string') return next(new HttpError(400, 'password must be a string'));
+    if ('key' in backupConfig && typeof backupConfig.key !== 'string') return next(new HttpError(400, 'key must be a string'));
     if (typeof backupConfig.format !== 'string') return next(new HttpError(400, 'format must be a string'));
     if ('acceptSelfSignedCerts' in backupConfig && typeof backupConfig.acceptSelfSignedCerts !== 'boolean') return next(new HttpError(400, 'format must be a boolean'));
 
-    if (typeof req.body.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string'));
+    if (typeof req.body.backupId !== 'string') return next(new HttpError(400, 'backupId must be a string or null'));
     if (typeof req.body.version !== 'string') return next(new HttpError(400, 'version must be a string'));
 
     if ('sysinfoConfig' in req.body && typeof req.body.sysinfoConfig !== 'object') return next(new HttpError(400, 'sysinfoConfig must be an object'));
@@ -122,7 +122,7 @@ function getStatus(req, res, next) {
 
         // check if Cloudron is not in setup state nor activated and let appstore know of the attempt
         if (!status.activated && !status.setup.active && !status.restore.active) {
-            appstore.trackBeginSetup();
+            appstore.trackBeginSetup(status.provider);
         }
     });
 }