5.3.4 changes

(cherry picked from commit bc6e652293)

CHANGES

@@ -1965,6 +1965,7 @@
 * better nginx config for higher loads
 * backups: add CIFS storage provider
 * backups: add SSHFS storage provider
+* backups: add NFS storage provider
 * s3: use vhost style
 * Fix crash when redis config was set
 * Update schedule was unselected in the UI
@@ -1978,4 +1979,37 @@
 * mail: make authentication case insensitive
 * Fix timeout issues in postgresql and mysql addon
 * Do not count stopped apps for memory use
+* LDAP group synchronization
+
+[5.3.1]
+* better nginx config for higher loads
+* backups: add CIFS storage provider
+* backups: add SSHFS storage provider
+* backups: add NFS storage provider
+* s3: use vhost style
+* Fix crash when redis config was set
+* Update schedule was unselected in the UI
+* cloudron-setup: --provider is now optional
+* show warning for unstable updates
+* add forumUrl to app manifest
+* postgresql: add unaccent extension for peertube
+* mail: Add Auto-Submitted header to NDRs
+* backups: ensure that the latest backup of installed apps is always preserved
+* add nginx logs
+* mail: make authentication case insensitive
+* Fix timeout issues in postgresql and mysql addon
+* Do not count stopped apps for memory use
+* LDAP group synchronization
+
+[5.3.2]
+* Do not install sshfs package
+* 'provider' is not required anymore in various API calls
+* redis: Set maxmemory and maxmemory-policy
+* Add mlock capability to manifest (for vault app)
+
+[5.3.3]
+* Fix issue where some postinstall messages where causing angular to infinite loop
+
+[5.3.4]
+* Fix issue in database error handling
 

package-lock.json

@@ -741,9 +741,9 @@
       }
     },
     "cloudron-manifestformat": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.3.0.tgz",
-      "integrity": "sha512-KMHTtR/oRnMzqTUzY1706xpYA1PrvmwIenC8HrigJhOYQdeMLv7egg1Eg0QIB1bfxKIhb+1Zc2i48HKE7MQGkg==",
+      "version": "5.4.0",
+      "resolved": "https://registry.npmjs.org/cloudron-manifestformat/-/cloudron-manifestformat-5.4.0.tgz",
+      "integrity": "sha512-MpgAMpBm3k14bH3lLaCUzcBtgC458Qx75blORHqTxJ83aGJp4P7+YYM/ABVGHVD0842OcR3JvQlCUT7+4cs6Cg==",
       "requires": {
         "cron": "^1.8.2",
         "java-packagename-regex": "^1.0.0",

package.json

@@ -20,7 +20,7 @@
     "async": "^2.6.3",
     "aws-sdk": "^2.685.0",
     "body-parser": "^1.19.0",
-    "cloudron-manifestformat": "^5.3.0",
+    "cloudron-manifestformat": "^5.4.0",
     "connect": "^3.7.0",
     "connect-lastmile": "^2.0.0",
     "connect-timeout": "^1.9.0",

scripts/cloudron-setup

@@ -159,7 +159,7 @@ if [[ "${initBaseImage}" == "true" ]]; then
     echo ""
 fi
 
-# NOTE: this install script only supports 4.2 and above
+# The provider flag is still used for marketplace images
 echo "=> Installing version ${version} (this takes some time) ..."
 mkdir -p /etc/cloudron
 echo "${provider}" > /etc/cloudron/PROVIDER

scripts/cloudron-support

@@ -112,7 +112,7 @@ if [[ "${enableSSH}" == "true" ]]; then
     permit_root_login=$(grep -q ^PermitRootLogin.*yes /etc/ssh/sshd_config && echo "yes" || echo "no")
 
     # support.js uses similar logic
-    if $(grep -q "ec2\|lightsail\|ami" /etc/cloudron/PROVIDER); then
+    if [[ -d /home/ubuntu ]]; then
         ssh_user="ubuntu"
         keys_file="/home/ubuntu/.ssh/authorized_keys"
     else

scripts/installer.sh

@@ -92,9 +92,6 @@ if [[ ${try} -eq 10 ]]; then
     exit 4
 fi
 
-echo "==> installer: ensure sshfs is installed"
-apt install -y sshfs
-
 echo "==> installer: downloading new addon images"
 images=$(node -e "var i = require('${box_src_tmp_dir}/src/infra_version.js'); console.log(i.baseImages.map(function (x) { return x.tag; }).join(' '), Object.keys(i.images).map(function (x) { return i.images[x].tag; }).join(' '));")
 

src/apps.js

@@ -427,7 +427,7 @@ function removeInternalFields(app) {
 function removeRestrictedFields(app) {
     return _.pick(app,
         'id', 'appStoreId', 'installationState', 'error', 'runState', 'health', 'taskId', 'alternateDomains', 'sso',
-        'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label');
+        'location', 'domain', 'fqdn', 'manifest', 'portBindings', 'iconUrl', 'creationTime', 'ts', 'tags', 'label', 'enableBackup');
 }
 
 function getIconUrlSync(app) {

src/appstore.js

@@ -11,7 +11,6 @@ exports = module.exports = {
     trackFinishedSetup: trackFinishedSetup,
 
     registerWithLoginCredentials: registerWithLoginCredentials,
-    registerWithLicense: registerWithLicense,
 
     purchaseApp: purchaseApp,
     unpurchaseApp: unpurchaseApp,
@@ -310,7 +309,6 @@ function sendAliveStatus(callback) {
         var data = {
             version: constants.VERSION,
             adminFqdn: settings.adminFqdn(),
-            provider: settings.provider(),
             backendSettings: backendSettings,
             machine: {
                 cpus: os.cpus(),
@@ -453,16 +451,14 @@ function registerCloudron(data, callback) {
 
 // This works without a Cloudron token as this Cloudron was not yet registered
 let gBeginSetupAlreadyTracked = false;
-function trackBeginSetup(provider) {
-    assert.strictEqual(typeof provider, 'string');
-
+function trackBeginSetup() {
     // avoid browser reload double tracking, not perfect since box might restart, but covers most cases and is simple
     if (gBeginSetupAlreadyTracked) return;
     gBeginSetupAlreadyTracked = true;
 
     const url = `${settings.apiServerOrigin()}/api/v1/helper/setup_begin`;
 
-    superagent.post(url).send({ provider }).timeout(30 * 1000).end(function (error, result) {
+    superagent.post(url).send({}).timeout(30 * 1000).end(function (error, result) {
         if (error && !error.response) return console.error(error.message);
         if (result.statusCode !== 200) return console.error(error.message);
     });
@@ -480,21 +476,6 @@ function trackFinishedSetup(domain) {
     });
 }
 
-function registerWithLicense(license, domain, callback) {
-    assert.strictEqual(typeof license, 'string');
-    assert.strictEqual(typeof domain, 'string');
-    assert.strictEqual(typeof callback, 'function');
-
-    getCloudronToken(function (error, token) {
-        if (token) return callback(new BoxError(BoxError.CONFLICT, 'Cloudron is already registered'));
-
-        const provider = settings.provider();
-        const version = constants.VERSION;
-
-        registerCloudron({ license, domain, provider, version }, callback);
-    });
-}
-
 function registerWithLoginCredentials(options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -514,7 +495,7 @@ function registerWithLoginCredentials(options, callback) {
             login(options.email, options.password, options.totpToken || '', function (error, result) {
                 if (error) return callback(error);
 
-                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, provider: settings.provider(), version: constants.VERSION, purpose: options.purpose || '' }, callback);
+                registerCloudron({ domain: settings.adminDomain(), accessToken: result.accessToken, version: constants.VERSION, purpose: options.purpose || '' }, callback);
             });
         });
     });

src/cloudron.js

@@ -139,7 +139,6 @@ function getConfig(callback) {
             mailFqdn: settings.mailFqdn(),
             version: constants.VERSION,
             isDemo: settings.isDemo(),
-            provider: settings.provider(),
             cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
             footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
             features: appstore.getFeatures()

src/database.js

@@ -22,8 +22,7 @@ var assert = require('assert'),
     once = require('once'),
     util = require('util');
 
-var gConnectionPool = null,
-    gDefaultConnection = null;
+var gConnectionPool = null;
 
 const gDatabase = {
     hostname: '127.0.0.1',
@@ -43,66 +42,37 @@ function initialize(callback) {
         gDatabase.hostname = require('child_process').execSync('docker inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" mysql-server').toString().trim();
     }
 
+    // https://github.com/mysqljs/mysql#pool-options
     gConnectionPool  = mysql.createPool({
-        connectionLimit: 5, // this has to be > 1 since we store one connection as 'default'. the rest for transactions
+        connectionLimit: 5,
         host: gDatabase.hostname,
         user: gDatabase.username,
         password: gDatabase.password,
         port: gDatabase.port,
         database: gDatabase.name,
         multipleStatements: false,
+        waitForConnections: true, // getConnection() will wait until a connection is avaiable
         ssl: false,
         timezone: 'Z' // mysql follows the SYSTEM timezone. on Cloudron, this is UTC
     });
 
     gConnectionPool.on('connection', function (connection) {
+        // connection objects are re-used. so we have to attach to the event here (once) to prevent crash
+        // note the pool also has an 'acquire' event but that is called whenever we do a getConnection()
+        connection.on('error', (error) => debug(`Connection ${connection.threadId} error: ${error.message} ${error.code}`));
+
         connection.query('USE ' + gDatabase.name);
         connection.query('SET SESSION sql_mode = \'strict_all_tables\'');
     });
 
-    reconnect(callback);
+    callback(null);
 }
 
 function uninitialize(callback) {
-    if (gConnectionPool) {
-        gConnectionPool.end(callback);
-        gDefaultConnection = null;
-        gConnectionPool = null;
-    } else {
-        callback(null);
-    }
-}
+    if (!gConnectionPool) return callback(null);
 
-function reconnect(callback) {
-    callback = callback ? once(callback) : function () {};
-
-    debug('reconnect: connecting to database');
-
-    gConnectionPool.getConnection(function (error, connection) {
-        if (error) {
-            debug(`reconnect: db connection error. ${error.message} fatal:${error.fatal} code:${error.code}. Will retry in 10 seconds`);
-            return setTimeout(reconnect.bind(null, callback), 10000);
-        }
-
-        debug('reconnect: connected to database');
-
-        connection.on('error', function (error) {
-            // by design, we catch all normal errors by providing callbacks.
-            // this function should be invoked only when we have no callbacks pending and we have a fatal error
-            assert(error.fatal, 'Non-fatal error on connection object');
-
-            debug(`reconnect: db connection error. ${error.message} fatal:${error.fatal} code:${error.code}. Will retry in 10 seconds`);
-
-            gDefaultConnection = null;
-
-            // This is most likely an issue an can cause double callbacks from reconnect()
-            setTimeout(reconnect.bind(null, callback), 10000);
-        });
-
-        gDefaultConnection = connection;
-
-        callback(null);
-    });
+    gConnectionPool.end(callback);
+    gConnectionPool = null;
 }
 
 function clear(callback) {
@@ -115,39 +85,14 @@ function clear(callback) {
     child_process.exec(cmd, callback);
 }
 
-function beginTransaction(callback) {
-    assert.strictEqual(typeof callback, 'function');
-
-    if (gConnectionPool === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No database connection pool.'));
-
-    gConnectionPool.getConnection(function (error, connection) {
-        if (error) return callback(error);
-
-        connection.beginTransaction(function (error) {
-            if (error) return callback(error);
-
-            return callback(null, connection);
-        });
-    });
-}
-
 function query() {
-    var args = Array.prototype.slice.call(arguments);
-    var callback = args[args.length - 1];
+    const args = Array.prototype.slice.call(arguments);
+    const callback = args[args.length - 1];
     assert.strictEqual(typeof callback, 'function');
 
-    if (gDefaultConnection === null) return callback(new BoxError(BoxError.DATABASE_ERROR, 'No connection to database'));
+    if (constants.TEST && !gConnectionPool) return callback(new BoxError(BoxError.DATABASE_ERROR, 'database.js not initialized'));
 
-    gDefaultConnection.query.apply(gDefaultConnection, args);
-}
-
-function rollback(connection, transactionError, callback) {
-    connection.rollback(function (error) {
-        if (error) debug('rollback: error when rolling back', error);
-
-        connection.release();
-        callback(transactionError);
-    });
+    gConnectionPool.query.apply(gConnectionPool, args); // this is same as getConnection/query/release
 }
 
 function transaction(queries, callback) {
@@ -156,21 +101,26 @@ function transaction(queries, callback) {
 
     callback = once(callback);
 
-    beginTransaction(function (error, connection) {
+    gConnectionPool.getConnection(function (error, connection) {
         if (error) return callback(error);
 
-        connection.on('error', callback);
+        const releaseConnection = (error) => { connection.release(); callback(error); };
 
-        async.mapSeries(queries, function iterator(query, done) {
-            connection.query(query.query, query.args, done);
-        }, function seriesDone(error, results) {
-            if (error) return rollback(connection, error, callback);
+        connection.beginTransaction(function (error) {
+            if (error) return releaseConnection(error);
 
-            connection.commit(function (error) {
-                if (error) return rollback(connection, error, callback);
+            async.mapSeries(queries, function iterator(query, done) {
+                connection.query(query.query, query.args, done);
+            }, function seriesDone(error, results) {
+                if (error) return connection.rollback(() => releaseConnection(error));
 
-                connection.release();
-                callback(null, results);
+                connection.commit(function (error) {
+                    if (error) return connection.rollback(() => releaseConnection(error));
+
+                    connection.release();
+
+                    callback(null, results);
+                });
             });
         });
     });

src/docker.js

@@ -306,6 +306,7 @@ function createSubcontainer(app, name, cmd, options, callback) {
                 Dns: ['172.18.0.1'], // use internal dns
                 DnsSearch: ['.'], // use internal dns
                 SecurityOpt: [ 'apparmor=docker-cloudron-app' ],
+                CapAdd: [],
                 CapDrop: [ 'NET_RAW' ] // https://docs-stage.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities
             },
             NetworkingConfig: {
@@ -318,11 +319,8 @@ function createSubcontainer(app, name, cmd, options, callback) {
         };
 
         var capabilities = manifest.capabilities || [];
-        if (capabilities.includes('net_admin')) {
-            containerOptions.HostConfig.CapAdd = [
-                'NET_ADMIN', 'NET_RAW'
-            ];
-        }
+        if (capabilities.includes('net_admin')) containerOptions.HostConfig.CapAdd.push('NET_ADMIN', 'NET_RAW');
+        if (capabilities.includes('mlock')) containerOptions.HostConfig.CapAdd.push('IPC_LOCK'); // mlock prevents swapping
 
         containerOptions = _.extend(containerOptions, options);
 

src/externalldap.js

@@ -22,6 +22,7 @@ var assert = require('assert'),
     debug = require('debug')('box:externalldap'),
     groups = require('./groups.js'),
     ldap = require('ldapjs'),
+    once = require('once'),
     settings = require('./settings.js'),
     tasks = require('./tasks.js'),
     users = require('./users.js');
@@ -60,18 +61,33 @@ function getClient(externalLdapConfig, callback) {
     assert.strictEqual(typeof externalLdapConfig, 'object');
     assert.strictEqual(typeof callback, 'function');
 
+    // ensure we only callback once since we also have to listen to client.error events
+    callback = once(callback);
+
     // basic validation to not crash
     try { ldap.parseDN(externalLdapConfig.baseDn); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid baseDn')); }
     try { ldap.parseFilter(externalLdapConfig.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
 
+    var config = {
+        url: externalLdapConfig.url,
+        tlsOptions: {
+            rejectUnauthorized: externalLdapConfig.acceptSelfSignedCerts ? false : true
+        }
+    };
+
     var client;
     try {
-        client = ldap.createClient({ url: externalLdapConfig.url });
+        client = ldap.createClient(config);
     } catch (e) {
         if (e instanceof ldap.ProtocolError) return callback(new BoxError(BoxError.BAD_FIELD, 'url protocol is invalid'));
         return callback(new BoxError(BoxError.INTERNAL_ERROR, e));
     }
 
+    // ensure we don't just crash
+    client.on('error', function (error) {
+        callback(new BoxError(BoxError.EXTERNAL_ERROR, error));
+    });
+
     if (!externalLdapConfig.bindDn) return callback(null, client);
 
     client.bind(externalLdapConfig.bindDn, externalLdapConfig.bindPassword, function (error) {
@@ -220,6 +236,7 @@ function testConfig(config, callback) {
     try { ldap.parseFilter(config.filter); } catch (e) { return callback(new BoxError(BoxError.BAD_FIELD, 'invalid filter')); }
 
     if ('syncGroups' in config && typeof config.syncGroups !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'syncGroups must be a boolean'));
+    if ('acceptSelfSignedCerts' in config && typeof config.acceptSelfSignedCerts !== 'boolean') return callback(new BoxError(BoxError.BAD_FIELD, 'acceptSelfSignedCerts must be a boolean'));
 
     if (config.syncGroups) {
         if (!config.groupBaseDn) return callback(new BoxError(BoxError.BAD_FIELD, 'groupBaseDn must not be empty'));

src/infra_version.js

@@ -19,7 +19,7 @@ exports = module.exports = {
         'mysql': { repo: 'cloudron/mysql', tag: 'cloudron/mysql:2.3.1@sha256:c1145d43c8a912fe6f5a5629a4052454a4aa6f23391c1efbffeec9d12d72a256' },
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:2.2.1@sha256:430f3e8b700327d4afa03a7b4e10a8b5544f171e0946ead8cdc5b67ee32db8e4' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:2.2.0@sha256:205486ff0f6bf6854610572df401cf3651bc62baf28fd26e9c5632497f10c2cb' },
-        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.2.0@sha256:cfdcc1a54cf29818cac99eacedc2ecf04e44995be3d06beea11dcaa09d90ed8d' },
+        'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.3.0@sha256:0e31ec817e235b1814c04af97b1e7cf0053384aca2569570ce92bef0d95e94d2' },
         'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.9.4@sha256:0e169b97a0584a76197d2bbc039d8698bf93f815588b3b43c251bd83dd545465' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.3.0@sha256:b7bc1ca4f4d0603a01369a689129aa273a938ce195fe43d00d42f4f2d5212f50' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:1.1.0@sha256:0c1fe4dd6121900624dcb383251ecb0084c3810e095064933de671409d8d6d7b' }

src/provision.js

@@ -226,7 +226,6 @@ function getStatus(callback) {
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
                 webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
-                provider: settings.provider(),
                 cloudronName: allSettings[settings.CLOUDRON_NAME_KEY],
                 footer: allSettings[settings.FOOTER_KEY] || constants.FOOTER,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,

src/routes/provision.js

@@ -122,7 +122,7 @@ function getStatus(req, res, next) {
 
         // check if Cloudron is not in setup state nor activated and let appstore know of the attempt
         if (!status.activated && !status.setup.active && !status.restore.active) {
-            appstore.trackBeginSetup(status.provider);
+            appstore.trackBeginSetup();
         }
     });
 }

src/storage/filesystem.js

@@ -54,7 +54,7 @@ function checkPreconditions(apiConfig, dataLayout, callback) {
     assert(dataLayout instanceof DataLayout, 'dataLayout must be a DataLayout');
     assert.strictEqual(typeof callback, 'function');
 
-    // TODO check filesystem is mounted for sshfs and cifs so we don't write into the actual folder on disk
+    // Check filesystem is mounted so we don't write into the actual folder on disk
     if (apiConfig.provider === PROVIDER_SSHFS || apiConfig.provider === PROVIDER_CIFS || apiConfig.provider === PROVIDER_NFS) {
         const mounts = safe.fs.readFileSync('/proc/mounts', 'utf8');
         const mountInfo = mounts.split('\n').filter(function (l) { return l.indexOf(apiConfig.mountPoint) !== -1; })[0];

src/support.js

@@ -12,7 +12,7 @@ let assert = require('assert'),
     once = require('once'),
     path = require('path'),
     paths = require('./paths.js'),
-    settings = require('./settings.js'),
+    safe = require('safetydance'),
     shell = require('./shell.js');
 
 // the logic here is also used in the cloudron-support tool
@@ -24,7 +24,7 @@ function sshInfo() {
     if (constants.TEST) {
         filePath = path.join(paths.baseDir(), 'authorized_keys');
         user = process.getuid();
-    } else if (settings.provider() === 'ec2' || settings.provider() === 'lightsail' || settings.provider() === 'ami') {
+    } else if (safe.fs.existsSync('/home/ubuntu')) { // yellowtent user won't have access to anything deeper
         filePath = '/home/ubuntu/.ssh/authorized_keys';
         user = 'ubuntu';
     } else {

src/test/appstore-test.js

@@ -67,7 +67,6 @@ describe('Appstore', function () {
             .post(`/api/v1/alive?accessToken=${APPSTORE_TOKEN}`, function (body) {
                 expect(body.version).to.be.a('string');
                 expect(body.adminFqdn).to.be.a('string');
-                expect(body.provider).to.be.a('string');
                 expect(body.backendSettings).to.be.an('object');
                 expect(body.backendSettings.backupConfig).to.be.an('object');
                 expect(body.backendSettings.backupConfig.provider).to.be.a('string');