Add changes

Fixes #682

CHANGES

@@ -1940,4 +1940,12 @@
 * Fix app disk graphs
 * restart apps on addon container change
 
+[5.2.2]
+* regression: import UI
+* Mbps -> MBps
+* Remove verbose logs
+* Set dmode in tar extract
+* mail: fix crash in audit logs
+* import: fix crash because encryption is unset
+* create redis with the correct label
 

src/addons.js

@@ -1723,10 +1723,10 @@ function startRedis(existingInfra, callback) {
     const tag = infra.images.redis.tag;
     const upgrading = existingInfra.version !== 'none' && requiresUpgrade(existingInfra.images.redis.tag, tag);
 
-    appdb.getAll(function (error, apps) {
+    apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        async.eachSeries(apps, function iterator (app, iteratorCallback) {
+        async.eachSeries(allApps, function iterator (app, iteratorCallback) {
             if (!('redis' in app.manifest.addons)) return iteratorCallback(); // app doesn't use the addon
 
             setupRedis(app, app.manifest.addons.redis, iteratorCallback);

src/apphealthmonitor.js

@@ -73,7 +73,6 @@ function checkAppHealth(app, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     if (app.installationState !== apps.ISTATE_INSTALLED || app.runState !== apps.RSTATE_RUNNING) {
-        debugApp(app, 'skipped. istate:%s rstate:%s', app.installationState, app.runState);
         return callback(null);
     }
 
@@ -103,10 +102,8 @@ function checkAppHealth(app, callback) {
             .timeout(HEALTHCHECK_INTERVAL)
             .end(function (error, res) {
                 if (error && !error.response) {
-                    debugApp(app, 'not alive (network error): %s', error.message);
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                 } else if (res.statusCode >= 400) { // 2xx and 3xx are ok
-                    debugApp(app, 'not alive : %s', error || res.status);
                     setHealth(app, apps.HEALTH_UNHEALTHY, callback);
                 } else {
                     setHealth(app, apps.HEALTH_HEALTHY, callback);
@@ -180,18 +177,16 @@ function processDockerEvents(intervalSecs, callback) {
 function processApp(callback) {
     assert.strictEqual(typeof callback, 'function');
 
-    apps.getAll(function (error, result) {
+    apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
-        async.each(result, checkAppHealth, function (error) {
+        async.each(allApps, checkAppHealth, function (error) {
             if (error) console.error(error);
 
-            const alive = result
-                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; })
-                .map(a => a.fqdn)
-                .join(', ');
+            const alive = allApps
+                .filter(function (a) { return a.installationState === apps.ISTATE_INSTALLED && a.runState === apps.RSTATE_RUNNING && a.health === apps.HEALTH_HEALTHY; });
 
-            debug('apps alive: [%s]', alive);
+            debug(`app health: ${alive.length} alive / ${allApps.length - alive.length} dead`);
 
             callback(null);
         });

src/apps.js

@@ -1542,9 +1542,13 @@ function importApp(app, data, auditSource, callback) {
     testBackupConfig(function (error) {
         if (error) return callback(error);
 
-        if (backupConfig && 'password' in backupConfig) {
-            backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
-            delete backupConfig.password;
+        if (backupConfig) {
+            if ('password' in backupConfig) {
+                backupConfig.encryption = backups.generateEncryptionKeysSync(backupConfig.password);
+                delete backupConfig.password;
+            } else {
+                backupConfig.encryption = null;
+            }
         }
 
         const restoreConfig = { backupId, backupFormat, backupConfig };

src/backups.js

@@ -519,8 +519,8 @@ function sync(backupConfig, backupId, dataLayout, progressCallback, callback) {
                 }); // ignore error if file disappears
                 stream.on('progress', function (progress) {
                     const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0Mbps looks wrong
-                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}Mbps` }); // 0M@0Mbps looks wrong
+                    if (!transferred && !speed) return progressCallback({ message: `Uploading ${task.path}` }); // 0M@0MBps looks wrong
+                    progressCallback({ message: `Uploading ${task.path}: ${transferred}M@${speed}MBps` }); // 0M@0MBps looks wrong
                 });
                 api(backupConfig.provider).upload(backupConfig, backupFilePath, stream, function (error) {
                     debug(error ? `Error uploading ${task.path} try ${retryCount}: ${error.message}` : `Uploaded ${task.path}`);
@@ -618,8 +618,8 @@ function upload(backupId, format, dataLayoutString, progressCallback, callback)
 
                         tarStream.on('progress', function (progress) {
                             const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                            if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0Mbps looks wrong
-                            progressCallback({ message: `Uploading backup ${transferred}M@${speed}Mbps` });
+                            if (!transferred && !speed) return progressCallback({ message: 'Uploading backup' }); // 0M@0MBps looks wrong
+                            progressCallback({ message: `Uploading backup ${transferred}M@${speed}MBps` });
                         });
                         tarStream.on('error', retryCallback); // already returns BoxError
 
@@ -648,7 +648,8 @@ function tarExtract(inStream, dataLayout, encryption, callback) {
         map: function (header) {
             header.name = dataLayout.toLocalPath(header.name);
             return header;
-        }
+        },
+        dmode: 500 // ensure directory is writable
     });
 
     const emitError = once((error) => {
@@ -759,8 +760,8 @@ function downloadDir(backupConfig, backupFilePath, dataLayout, progressCallback,
 
                     destStream.on('progress', function (progress) {
                         const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0Mbps looks wrong
-                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}Mbps` });
+                        if (!transferred && !speed) return progressCallback({ message: `Downloading ${entry.fullPath}` }); // 0M@0MBps looks wrong
+                        progressCallback({ message: `Downloading ${entry.fullPath}: ${transferred}M@${speed}MBps` });
                     });
                     destStream.on('error', closeAndRetry);
 
@@ -804,8 +805,8 @@ function download(backupConfig, backupId, format, dataLayout, progressCallback,
 
                     ps.on('progress', function (progress) {
                         const transferred = Math.round(progress.transferred/1024/1024), speed = Math.round(progress.speed/1024/1024);
-                        if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0Mbps looks wrong
-                        progressCallback({ message: `Downloading ${transferred}M@${speed}Mbps` });
+                        if (!transferred && !speed) return progressCallback({ message: 'Downloading backup' }); // 0M@0MBps looks wrong
+                        progressCallback({ message: `Downloading ${transferred}M@${speed}MBps` });
                     });
                     ps.on('error', retryCallback);
                     ps.on('done', retryCallback);

src/docker.js

@@ -53,12 +53,6 @@ const CLEARVOLUME_CMD = path.join(__dirname, 'scripts/clearvolume.sh'),
 const DOCKER_SOCKET_PATH = '/var/run/docker.sock';
 const gConnection = new Docker({ socketPath: DOCKER_SOCKET_PATH });
 
-function debugApp(app) {
-    assert(typeof app === 'object');
-
-    debug(app.fqdn + ' ' + util.format.apply(util, Array.prototype.slice.call(arguments, 1)));
-}
-
 function testRegistryConfig(auth, callback) {
     assert.strictEqual(typeof auth, 'object');
     assert.strictEqual(typeof callback, 'function');
@@ -332,8 +326,6 @@ function createSubcontainer(app, name, cmd, options, callback) {
 
         containerOptions = _.extend(containerOptions, options);
 
-        debugApp(app, 'Creating container for %s', app.manifest.dockerImage);
-
         gConnection.createContainer(containerOptions, function (error, container) {
             if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 
@@ -351,7 +343,6 @@ function startContainer(containerId, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var container = gConnection.getContainer(containerId);
-    debug('Starting container %s', containerId);
 
     container.start(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -367,7 +358,6 @@ function restartContainer(containerId, callback) {
     assert.strictEqual(typeof callback, 'function');
 
     var container = gConnection.getContainer(containerId);
-    debug('Restarting container %s', containerId);
 
     container.restart(function (error) {
         if (error && error.statusCode === 404) return callback(new BoxError(BoxError.NOT_FOUND));
@@ -388,7 +378,6 @@ function stopContainer(containerId, callback) {
     }
 
     var container = gConnection.getContainer(containerId);
-    debug('Stopping container %s', containerId);
 
     var options = {
         t: 10 // wait for 10 seconds before killing it
@@ -397,13 +386,9 @@ function stopContainer(containerId, callback) {
     container.stop(options, function (error) {
         if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error stopping container:' + error.message));
 
-        debug('Waiting for container ' + containerId);
-
-        container.wait(function (error, data) {
+        container.wait(function (error/*, data */) {
             if (error && (error.statusCode !== 304 && error.statusCode !== 404)) return callback(new BoxError(BoxError.DOCKER_ERROR, 'Error waiting on container:' + error.message));
 
-            debug('Container %s stopped with status code [%s]', containerId, data ? String(data.StatusCode) : '');
-
             return callback(null);
         });
     });
@@ -413,8 +398,6 @@ function deleteContainer(containerId, callback) {
     assert(!containerId || typeof containerId === 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('deleting container %s', containerId);
-
     if (containerId === null) return callback(null);
 
     var container = gConnection.getContainer(containerId);
@@ -441,8 +424,6 @@ function deleteContainers(appId, options, callback) {
     assert.strictEqual(typeof options, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('deleting containers of %s', appId);
-
     let labels = [ 'appId=' + appId ];
     if (options.managedOnly) labels.push('isCloudronManaged=true');
 
@@ -459,8 +440,6 @@ function stopContainers(appId, callback) {
     assert.strictEqual(typeof appId, 'string');
     assert.strictEqual(typeof callback, 'function');
 
-    debug('Stopping containers of %s', appId);
-
     gConnection.listContainers({ all: 1, filters: JSON.stringify({ label: [ 'appId=' + appId ] }) }, function (error, containers) {
         if (error) return callback(new BoxError(BoxError.DOCKER_ERROR, error));
 

src/infra_version.js

@@ -20,7 +20,7 @@ exports = module.exports = {
         'postgresql': { repo: 'cloudron/postgresql', tag: 'cloudron/postgresql:2.1.1@sha256:6ea1bcf9b655d628230acda01d46cf21883b112111d89583c94138646895c14c' },
         'mongodb': { repo: 'cloudron/mongodb', tag: 'cloudron/mongodb:2.2.0@sha256:205486ff0f6bf6854610572df401cf3651bc62baf28fd26e9c5632497f10c2cb' },
         'redis': { repo: 'cloudron/redis', tag: 'cloudron/redis:2.2.0@sha256:cfdcc1a54cf29818cac99eacedc2ecf04e44995be3d06beea11dcaa09d90ed8d' },
-        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.9.2@sha256:d71d7fa5be9a578aef5f24aceaa0d6fe59ef1e8d2858263669ddf58703d1a963' },
+        'mail': { repo: 'cloudron/mail', tag: 'cloudron/mail:2.9.3@sha256:2c062e8883cadae4b4c22553f9db141fc441d6c627aa85e2a5ad71e3fcbf2b42' },
         'graphite': { repo: 'cloudron/graphite', tag: 'cloudron/graphite:2.3.0@sha256:b7bc1ca4f4d0603a01369a689129aa273a938ce195fe43d00d42f4f2d5212f50' },
         'sftp': { repo: 'cloudron/sftp', tag: 'cloudron/sftp:1.1.0@sha256:0c1fe4dd6121900624dcb383251ecb0084c3810e095064933de671409d8d6d7b' }
     }

src/scheduler.js

@@ -23,8 +23,6 @@ function sync(callback) {
 
     callback = callback || NOOP_CALLBACK;
 
-    debug('sync: synchronizing global state with installed app state');
-
     apps.getAll(function (error, allApps) {
         if (error) return callback(error);
 
@@ -49,8 +47,6 @@ function sync(callback) {
                     return iteratorDone(); // nothing changed
                 }
 
-                debug(`sync: app ${app.fqdn} changed`);
-
                 stopJobs(app.id, appState, function (error) {
                     if (error) debug(`sync: error stopping jobs of ${app.fqdn} : ${error.message}`);
 
@@ -67,8 +63,6 @@ function sync(callback) {
                     iteratorDone();
                 });
             });
-
-            debug('sync: done');
         });
     });
 }
@@ -92,8 +86,6 @@ function stopJobs(appId, appState, callback) {
     assert.strictEqual(typeof appState, 'object');
     assert.strictEqual(typeof callback, 'function');
 
-    debug(`stopJobs: stopping jobs of ${appId}`);
-
     if (!appState) return callback();
 
     async.eachSeries(Object.keys(appState.schedulerConfig), function (taskName, iteratorDone) {
@@ -109,8 +101,6 @@ function createCronJobs(app, schedulerConfig) {
     assert.strictEqual(typeof app, 'object');
     assert(schedulerConfig && typeof schedulerConfig === 'object');
 
-    debug(`createCronJobs: creating cron jobs for app ${app.fqdn}`);
-
     var jobs = { };
 
     Object.keys(schedulerConfig).forEach(function (taskName) {
@@ -120,8 +110,6 @@ function createCronJobs(app, schedulerConfig) {
 
         var cronTime = (constants.TEST ? '*/5 ' : `${randomSecond} `) + task.schedule; // time ticks faster in tests
 
-        debug(`createCronJobs: ${app.fqdn} task ${taskName} scheduled at ${cronTime} with cmd ${task.command}`);
-
         var cronJob = new CronJob({
             cronTime: cronTime, // at this point, the pattern has been validated
             onTick: runTask.bind(null, app.id, taskName), //  put the app id in closure, so we don't use the outdated app object by mistake
@@ -143,8 +131,6 @@ function runTask(appId, taskName, callback) {
 
     callback = callback || NOOP_CALLBACK;
 
-    debug(`runTask: running task ${taskName} of ${appId}`);
-
     apps.get(appId, function (error, app) {
         if (error) return callback(error);
 
@@ -158,20 +144,13 @@ function runTask(appId, taskName, callback) {
         docker.inspectByName(containerName, function (err, data) {
             if (!err && data && data.State.Running === true) {
                 const jobStartTime = new Date(data.State.StartedAt); // iso 8601
-                if (new Date() - jobStartTime < JOB_MAX_TIME) {
-                    debug(`runTask: skipped task ${taskName} of app ${app.fqdn} since it was started at ${jobStartTime}`);
-                    return callback();
-                }
+                if (new Date() - jobStartTime < JOB_MAX_TIME) return callback();
             }
 
-            debug(`runTask: removing any old task ${taskName} of app ${app.fqdn}`);
-
             killContainer(containerName, function (error) {
                 if (error) return callback(error);
                 const cmd = gState[appId].schedulerConfig[taskName].command;
 
-                debug(`runTask: starting task ${taskName} of app ${app.fqdn} with cmd ${cmd}`);
-
                 // NOTE: if you change container name here, fix addons.js to return correct container names
                 docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', cmd ], { } /* options */, function (error, container) {
                     if (error) return callback(error);