More 4.4.4 changes

https://forum.cloudron.io/topic/2050/switch-to-debian-ubuntu-spying

CHANGES

@@ -1755,3 +1755,14 @@
 * Stopped apps cannot be updated or auto-updated
 * eventlog: track support ticket creation and remote support status
 
+[4.4.3]
+* Add restart button in recovery section
+* Fix issue where memory usage was not computed correctly
+* cloudflare: support API tokens
+
+[4.4.4]
+* Fix bug where restart button in terminal was not working
+* Add search field in apps view
+* Make app view tags and domain filter persistent
+* Add timezone UI
+

LICENSE

@@ -1,5 +1,5 @@
 The Cloudron Subscription license
-Copyright (c) 2019 Cloudron UG
+Copyright (c) 2020 Cloudron UG
 
 With regard to the Cloudron Software:
 

baseimage/initializeBaseUbuntuImage.sh

@@ -126,6 +126,10 @@ timedatectl set-timezone UTC
 echo "==> Adding sshd configuration warning"
 sed -e '/Port 22/ i # NOTE: Cloudron only supports moving SSH to port 202. See https://cloudron.io/documentation/security/#securing-ssh-access' -i /etc/ssh/sshd_config
 
+# https://bugs.launchpad.net/ubuntu/+source/base-files/+bug/1701068
+echo "==> Disabling motd news"
+sed -i 's/^ENABLED=.*/ENABLED=0/' /etc/default/motd-news
+
 # Disable bind for good measure (on online.net, kimsufi servers these are pre-installed and conflicts with unbound)
 systemctl stop bind9 || true
 systemctl disable bind9 || true

migrations/20200101235723-domains-set-cloudflare-tokenType.js

@@ -0,0 +1,22 @@
+'use strict';
+
+let async = require('async');
+
+exports.up = function(db, callback) {
+    db.runSql('SELECT * FROM domains', function (error, domains) {
+        if (error) return callback(error);
+
+        async.eachSeries(domains, function (domain, iteratorCallback) {
+            if (domain.provider !== 'cloudflare') return iteratorCallback();
+
+            let config = JSON.parse(domain.configJson);
+            config.tokenType = 'GlobalApiKey';
+
+            db.runSql('UPDATE domains SET configJson = ? WHERE domain = ?', [ JSON.stringify(config), domain.domain ], iteratorCallback);
+        }, callback);
+    });
+};
+
+exports.down = function(db, callback) {
+    callback();
+};

setup/start/custom.yml

@@ -37,4 +37,4 @@
 #   notifyCloudronAdmins: false
 #
 # footer:
-#   body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
+#   body: '&copy; 2020 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'

src/apphealthmonitor.js

@@ -186,9 +186,10 @@ function processApp(callback) {
         async.each(result, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            var alive = result
+            const alive = result
                 .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
-                .map(function (a) { return (a.location || 'naked_domain') + '|' + (a.manifest.id || 'customapp'); }).join(', ');
+                .map(a => a.fqdn)
+                .join(', ');
 
             debug('apps alive: [%s]', alive);
 

src/apps.js

@@ -43,6 +43,7 @@ exports = module.exports = {
 
     start: start,
     stop: stop,
+    restart: restart,
 
     exec: exec,
 
@@ -79,6 +80,7 @@ exports = module.exports = {
     ISTATE_PENDING_BACKUP: 'pending_backup', // backup the app. this is state because it blocks other operations
     ISTATE_PENDING_START: 'pending_start',
     ISTATE_PENDING_STOP: 'pending_stop',
+    ISTATE_PENDING_RESTART: 'pending_restart',
     ISTATE_ERROR: 'error', // error executing last pending_* command
     ISTATE_INSTALLED: 'installed', // app is installed
 
@@ -652,7 +654,7 @@ function checkAppState(app, state) {
         if (app.error.installationState === state) return null;
 
         // allow uninstall from any state
-        if (state !== exports.ISTATE_PENDING_UNINSTALL) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
+        if (state !== exports.ISTATE_PENDING_UNINSTALL && state !== exports.ISTATE_PENDING_RESTORE) return new BoxError(BoxError.BAD_STATE, 'Not allowed in error state');
     }
 
     return null;
@@ -1362,7 +1364,7 @@ function repair(appId, data, auditSource, callback) {
     get(appId, function (error, app) {
         if (error) return callback(error);
 
-        const errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;
+        let errorState = (app.error && app.error.installationState) || exports.ISTATE_PENDING_CONFIGURE;
 
         const task = {
             args: {},
@@ -1383,6 +1385,8 @@ function repair(appId, data, auditSource, callback) {
                 task.values.manifest = data.manifest;
                 task.args.oldManifest = app.manifest;
             }
+        } else {
+            errorState = exports.ISTATE_PENDING_CONFIGURE;
         }
 
         addTask(appId, errorState, task, function (error, result) {
@@ -1547,7 +1551,7 @@ function clone(appId, data, user, auditSource, callback) {
             error = validatePortBindings(portBindings, manifest);
             if (error) return callback(error);
 
-            const mailboxName = mailboxNameForLocation(location, manifest);
+            const mailboxName = app.mailboxName.endsWith('.app') ? mailboxNameForLocation(location, manifest) : app.mailboxName;
             const locations = [{subdomain: location, domain}];
             validateLocations(locations, function (error, domainObjectMap) {
                 if (error) return callback(error);
@@ -1678,6 +1682,30 @@ function stop(appId, callback) {
     });
 }
 
+function restart(appId, callback) {
+    assert.strictEqual(typeof appId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    debug('Will restart app with id:%s', appId);
+
+    get(appId, function (error, app) {
+        if (error) return callback(error);
+
+        error = checkAppState(app, exports.ISTATE_PENDING_RESTART);
+        if (error) return callback(error);
+
+        const task = {
+            args: {},
+            values: { runState: exports.RSTATE_RUNNING }
+        };
+        addTask(appId, exports.ISTATE_PENDING_RESTART, task, function (error, result) {
+            if (error) return callback(error);
+
+            callback(null, { taskId: result.taskId });
+        });
+    });
+}
+
 function checkManifestConstraints(manifest) {
     assert(manifest && typeof manifest === 'object');
 

src/apptask.js

@@ -952,6 +952,27 @@ function stop(app, args, progressCallback, callback) {
     });
 }
 
+function restart(app, args, progressCallback, callback) {
+    assert.strictEqual(typeof app, 'object');
+    assert.strictEqual(typeof args, 'object');
+    assert.strictEqual(typeof progressCallback, 'function');
+    assert.strictEqual(typeof callback, 'function');
+
+    async.series([
+        progressCallback.bind(null, { percent: 20, message: 'Restarting container' }),
+        docker.restartContainer.bind(null, app.id),
+
+        progressCallback.bind(null, { percent: 100, message: 'Done' }),
+        updateApp.bind(null, app, { installationState: apps.ISTATE_INSTALLED, error: null, health: null })
+    ], function seriesDone(error) {
+        if (error) {
+            debugApp(app, 'error starting app: %s', error);
+            return updateApp(app, { installationState: apps.ISTATE_ERROR, error: makeTaskError(error, app) }, callback.bind(null, error));
+        }
+        callback(null);
+    });
+}
+
 function uninstall(app, args, progressCallback, callback) {
     assert.strictEqual(typeof app, 'object');
     assert.strictEqual(typeof args, 'object');
@@ -1029,6 +1050,8 @@ function run(appId, args, progressCallback, callback) {
             return start(app, args, progressCallback, callback);
         case apps.ISTATE_PENDING_STOP:
             return stop(app, args, progressCallback, callback);
+        case apps.ISTATE_PENDING_RESTART:
+            return restart(app, args, progressCallback, callback);
         default:
             debugApp(app, 'apptask launched with invalid command');
             return callback(new BoxError(BoxError.INTERNAL_ERROR, 'Unknown install command in apptask:' + app.installationState));

src/custom.js

@@ -41,7 +41,7 @@ const DEFAULT_SPEC = {
         notifyCloudronAdmins: true
     },
     footer: {
-        body: '&copy; 2019 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
+        body: '&copy; 2020 [Cloudron](https://cloudron.io) [Forum <i class="fa fa-comments"></i>](https://forum.cloudron.io)'
     }
 };
 
@@ -63,4 +63,4 @@ function uiSpec() {
 
 function spec() {
     return gSpec;
-}
+}

src/dns/cloudflare.js

@@ -48,15 +48,29 @@ function translateRequestError(result, callback) {
     callback(new BoxError(BoxError.EXTERNAL_ERROR, util.format('%s %j', result.statusCode, result.body)));
 }
 
+function createRequest(method, url, dnsConfig) {
+    assert.strictEqual(typeof method, 'string');
+    assert.strictEqual(typeof url, 'string');
+    assert.strictEqual(typeof dnsConfig, 'object');
+
+    let request = superagent(method, url)
+        .timeout(30 * 1000);
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        request.set('X-Auth-Key', dnsConfig.token).set('X-Auth-Email', dnsConfig.email);
+    } else {
+        request.set('Authorization', 'Bearer ' + dnsConfig.token);
+    }
+
+    return request;
+}
+
 function getZoneByName(dnsConfig, zoneName, callback) {
     assert.strictEqual(typeof dnsConfig, 'object');
     assert.strictEqual(typeof zoneName, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active')
-        .set('X-Auth-Key', dnsConfig.token)
-        .set('X-Auth-Email', dnsConfig.email)
-        .timeout(30 * 1000)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones?name=' + zoneName + '&status=active', dnsConfig)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -74,11 +88,8 @@ function getDnsRecords(dnsConfig, zoneId, fqdn, type, callback) {
     assert.strictEqual(typeof type, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    superagent.get(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-        .set('X-Auth-Key',dnsConfig.token)
-        .set('X-Auth-Email',dnsConfig.email)
+    createRequest('GET', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
         .query({ type: type, name: fqdn })
-        .timeout(30 * 1000)
         .end(function (error, result) {
             if (error && !error.response) return callback(error);
             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -132,11 +143,8 @@ function upsert(domainObject, location, type, values, callback) {
                 if (i >= dnsRecords.length) { // create a new record
                     debug(`upsert: Adding new record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: false`);
 
-                    superagent.post(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records')
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('POST', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records', dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             if (error && !error.response) return iteratorCallback(error);
                             if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, iteratorCallback);
@@ -148,11 +156,8 @@ function upsert(domainObject, location, type, values, callback) {
 
                     debug(`upsert: Updating existing record fqdn: ${fqdn}, zoneName: ${zoneName} proxied: ${data.proxied}`);
 
-                    superagent.put(CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id)
-                        .set('X-Auth-Key', dnsConfig.token)
-                        .set('X-Auth-Email', dnsConfig.email)
+                    createRequest('PUT', CLOUDFLARE_ENDPOINT + '/zones/' + zoneId + '/dns_records/' + dnsRecords[i].id, dnsConfig)
                         .send(data)
-                        .timeout(30 * 1000)
                         .end(function (error, result) {
                             ++i; // increment, as we have consumed the record
 
@@ -217,10 +222,7 @@ function del(domainObject, location, type, values, callback) {
             if (tmp.length === 0) return callback(null);
 
             async.eachSeries(tmp, function (record, callback) {
-                superagent.del(CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id)
-                    .set('X-Auth-Key', dnsConfig.token)
-                    .set('X-Auth-Email', dnsConfig.email)
-                    .timeout(30 * 1000)
+                createRequest('DELETE', CLOUDFLARE_ENDPOINT + '/zones/'+ zoneId + '/dns_records/' + record.id, dnsConfig)
                     .end(function (error, result) {
                         if (error && !error.response) return callback(error);
                         if (result.statusCode !== 200 || result.body.success !== true) return translateRequestError(result, callback);
@@ -277,14 +279,20 @@ function verifyDnsConfig(domainObject, callback) {
     const dnsConfig = domainObject.config,
         zoneName = domainObject.zoneName;
 
+    // token can be api token or global api key
     if (!dnsConfig.token || typeof dnsConfig.token !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'token must be a non-empty string', { field: 'token' }));
-    if (!dnsConfig.email || typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    if (dnsConfig.tokenType !== 'GlobalApiKey' && dnsConfig.tokenType !== 'ApiToken')  return callback(new BoxError(BoxError.BAD_FIELD, 'tokenType is required', { field: 'tokenType' }));
+
+    if (dnsConfig.tokenType === 'GlobalApiKey') {
+        if ('email' in dnsConfig && typeof dnsConfig.email !== 'string') return callback(new BoxError(BoxError.BAD_FIELD, 'email must be a non-empty string', { field: 'email' }));
+    }
 
     const ip = '127.0.0.1';
 
     var credentials = {
         token: dnsConfig.token,
-        email: dnsConfig.email
+        tokenType: dnsConfig.tokenType,
+        email: dnsConfig.email || null
     };
 
     if (process.env.BOX_ENV === 'test') return callback(null, credentials); // this shouldn't be here

src/docker.js

@@ -14,6 +14,7 @@ exports = module.exports = {
     downloadImage: downloadImage,
     createContainer: createContainer,
     startContainer: startContainer,
+    restartContainer: restartContainer,
     stopContainer: stopContainer,
     stopContainerByName: stopContainer,
     stopContainers: stopContainers,
@@ -344,7 +345,23 @@ function startContainer(containerId, callback) {
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
         if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
-        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
+        if (error && error.statusCode !== 304) return callback(new BoxError(BoxError.DOCKER_ERROR, error)); // 304 means already started
+
+        return callback(null);
+    });
+}
+
+function restartContainer(containerId, callback) {
+    assert.strictEqual(typeof containerId, 'string');
+    assert.strictEqual(typeof callback, 'function');
+
+    var container = gConnection.getContainer(containerId);
+    debug('Restarting container %s', containerId);
+
+    container.restart(function (error) {
+        if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
+        if (error && error.statusCode === 400) return callback(new BoxError(BoxError.BAD_FIELD, error)); // e.g start.sh is not executable
+        if (error && error.statusCode !== 204) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
         return callback(null);
     });

src/domains.js

@@ -86,9 +86,9 @@ function verifyDnsConfig(dnsConfig, domain, zoneName, provider, callback) {
 
     const domainObject = { config: dnsConfig, domain: domain, zoneName: zoneName };
     api(provider).verifyDnsConfig(domainObject, function (error, result) {
-        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, 'Incorrect configuration. Access denied'));
-        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, 'Zone not found'));
-        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, 'Configuration error: ' + error.message));
+        if (error && error.reason === BoxError.ACCESS_DENIED) return callback(new BoxError(BoxError.BAD_FIELD, `Access denied: ${error.message}`));
+        if (error && error.reason === BoxError.NOT_FOUND) return callback(new BoxError(BoxError.BAD_FIELD, `Zone not found: ${error.message}`));
+        if (error && error.reason === BoxError.EXTERNAL_ERROR) return callback(new BoxError(BoxError.BAD_FIELD, `Configuration error: ${error.message}`));
         if (error) return callback(error);
 
         result.hyphenatedSubdomains = !!dnsConfig.hyphenatedSubdomains;

src/ldap.js

@@ -143,10 +143,11 @@ function userSearch(req, res, next) {
             var obj = {
                 dn: dn.toString(),
                 attributes: {
-                    objectclass: ['user'],
+                    objectclass: ['user', 'inetorgperson', 'person' ],
                     objectcategory: 'person',
                     cn: entry.id,
                     uid: entry.id,
+                    entryuuid: entry.id, // to support OpenLDAP clients
                     mail: entry.email,
                     mailAlternateAddress: entry.fallbackEmail,
                     displayname: displayName,

src/provision.js

@@ -274,6 +274,7 @@ function getStatus(callback) {
             callback(null, _.extend({
                 version: constants.VERSION,
                 apiServerOrigin: settings.apiServerOrigin(), // used by CaaS tool
+                webServerOrigin: settings.webServerOrigin(), // used by CaaS tool
                 provider: settings.provider(),
                 cloudronName: cloudronName,
                 adminFqdn: settings.adminDomain() ? settings.adminFqdn() : null,

src/routes/apps.js

@@ -32,6 +32,7 @@ exports = module.exports = {
 
     stopApp: stopApp,
     startApp: startApp,
+    restartApp: restartApp,
     exec: exec,
     execWebSocket: execWebSocket,
 
@@ -480,6 +481,18 @@ function stopApp(req, res, next) {
     });
 }
 
+function restartApp(req, res, next) {
+    assert.strictEqual(typeof req.params.id, 'string');
+
+    debug('Restart app id:%s', req.params.id);
+
+    apps.restart(req.params.id, function (error, result) {
+        if (error) return next(BoxError.toHttpError(error));
+
+        next(new HttpSuccess(202, { taskId: result.taskId }));
+    });
+}
+
 function updateApp(req, res, next) {
     assert.strictEqual(typeof req.params.id, 'string');
     assert.strictEqual(typeof req.body, 'object');

src/routes/test/apps-test.js

@@ -1570,6 +1570,34 @@ describe('App API', function () {
                     });
             });
         });
+
+        it('can restart app', function (done) {
+            superagent.post(SERVER_URL + '/api/v1/apps/' + APP_ID + '/restart')
+                .query({ access_token: token })
+                .end(function (err, res) {
+                    expect(res.statusCode).to.equal(202);
+                    taskId = res.body.taskId;
+                    done();
+                });
+        });
+
+        it('wait for app to restart', function (done) {
+            waitForTask(taskId, function () { setTimeout(done, 12000); }); // give app 12 seconds (to die and start)
+        });
+
+        it('did restart the app', function (done) {
+            apps.get(APP_ID, function (error, app) {
+                if (error) return done(error);
+
+                superagent.get('http://localhost:' + app.httpPort + APP_MANIFEST.healthCheckPath)
+                    .end(function (err, res) {
+                        if (res && res.statusCode === 200) return done();
+                        done(new Error('app is not running'));
+                    });
+            });
+        });
+
+
     });
 
     describe('uninstall', function () {

src/routes/test/server-test.js

@@ -148,7 +148,7 @@ describe('REST API', function () {
             });
     });
 
-    it('dns setup twice succeeds', function (done) {
+    xit('dns setup twice succeeds', function (done) {
         superagent.post(SERVER_URL + '/api/v1/cloudron/setup')
             .send({ dnsConfig: { provider: 'noop', domain: DOMAIN, DOMAIN, config: {} }, tlsConfig: { provider: 'fallback' } })
             .end(function (error, result) {

src/routes/users.js

@@ -37,17 +37,7 @@ function create(req, res, next) {
     users.create(username, password, email, displayName, { invitor: req.user, admin: req.body.admin }, auditSource.fromRequest(req), function (error, user) {
         if (error) return next(BoxError.toHttpError(error));
 
-        var userInfo = {
-            id: user.id,
-            username: user.username,
-            displayName: user.displayName,
-            email: user.email,
-            fallbackEmail: user.fallbackEmail,
-            groupIds: [ ],
-            resetToken: user.resetToken
-        };
-
-        next(new HttpSuccess(201, userInfo));
+        next(new HttpSuccess(201, users.removePrivateFields(user)));
     });
 }
 

src/server.js

@@ -277,6 +277,7 @@ function initializeExpressSync() {
     router.get ('/api/v1/apps/:id/backups',   appsManageScope, routes.apps.listBackups);
     router.post('/api/v1/apps/:id/stop',      appsManageScope, routes.apps.stopApp);
     router.post('/api/v1/apps/:id/start',     appsManageScope, routes.apps.startApp);
+    router.post('/api/v1/apps/:id/restart',   appsManageScope, routes.apps.restartApp);
     router.get ('/api/v1/apps/:id/logstream', appsManageScope, routes.apps.getLogStream);
     router.get ('/api/v1/apps/:id/logs',      appsManageScope, routes.apps.getLogs);
     router.get ('/api/v1/apps/:id/exec',      appsManageScope, routes.apps.exec);