retry apply of platform config

it seems that scaling down addons can fail at times. this can happen
if the kernel is busy swapping things etc. so, we do not block the
platform startup for this operation.

Nov 13 05:20:00 localhost dockerd[28831]: time="2018-11-13T05:20:00.365056059Z" level=error msg="Handler for POST /v1.37/containers/mongodb/update returned error: Cannot update container 6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f: docker-runc did not terminate sucessfully: failed to write 419430400 to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f/memory.memsw.limit_in_bytes: device or resource busy\n: unknown"
Nov 13 05:20:00 localhost node[5486]: box:shell updatemongodb (stderr): Error response from daemon: Cannot update container 6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f: docker-runc did not terminate sucessfully: failed to write 419430400 to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f/memory.memsw.limit_in_bytes: device or resource busy

src/addons.js

@@ -43,7 +43,6 @@ var accesscontrol = require('./accesscontrol.js'),
     rimraf = require('rimraf'),
     safe = require('safetydance'),
     semver = require('semver'),
-    settings = require('./settings.js'),
     shell = require('./shell.js'),
     request = require('request'),
     util = require('util');
@@ -355,6 +354,8 @@ function updateAddonConfig(platformConfig, callback) {
         mail: Math.max((1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 128, 256) * 1024 * 1024
     };
 
+    debug('updateAddonConfig: %j', platformConfig);
+
     // TODO: this should possibly also rollback memory to default
     async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb' ], function iterator(containerName, iteratorCallback) {
         const containerConfig = platformConfig[containerName];
@@ -399,16 +400,7 @@ function startAddons(existingInfra, callback) {
         debug('startAddons: existing infra. incremental addon create %j', startFuncs.map(function (f) { return f.name; }));
     }
 
-    async.series(startFuncs, function (error) {
+    async.series(startFuncs, callback);
-        if (error) return callback(error);
-
-        // once addons are started/imported, scale them back
-        settings.getPlatformConfig(function (error, platformConfig) {
-            if (error) return callback(error);
-
-            updateAddonConfig(platformConfig, callback);
-        });
-    });
 }
 
 function getEnvironment(app, callback) {

src/platform.js

@@ -25,6 +25,7 @@ var addons = require('./addons.js'),
     paths = require('./paths.js'),
     reverseProxy = require('./reverseproxy.js'),
     safe = require('safetydance'),
+    settings = require('./settings.js'),
     shell = require('./shell.js'),
     taskmanager = require('./taskmanager.js'),
     _ = require('underscore');
@@ -82,9 +83,27 @@ function stop(callback) {
 }
 
 function onPlatformReady() {
-    debug('onPlatformReady: resuming task manager');
+    debug('onPlatformReady: platform is ready');
     exports._isReady = true;
     taskmanager.resumeTasks();
+    applyPlatformConfig(NOOP_CALLBACK);
+}
+
+function applyPlatformConfig(callback) {
+    // scale back db containers, if possible. this is retried because updating memory constraints can fail
+    // with failed to write to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/xx/memory.memsw.limit_in_bytes: device or resource busy
+
+    async.retry({ times: 10, interval: 5 * 60 * 1000 }, function (retryCallback) {
+        settings.getPlatformConfig(function (error, platformConfig) {
+            if (error) return retryCallback(error);
+
+            addons.updateAddonConfig(platformConfig, function (error) {
+                if (error) debug('Error updating addons. Will rety in 5 minutes', platformConfig, error);
+
+                retryCallback(error);
+            });
+        });
+    }, callback);
 }
 
 function pruneInfraImages(callback) {

src/shell.js

@@ -38,7 +38,7 @@ function exec(tag, file, args, options, callback) {
 
     callback = once(callback); // exit may or may not be called after an 'error'
 
-    debug(tag + ' execFile: %s', file); // do not dump args as it might have sensitive info
+    debug(tag + ' execFile: %s %s', file, args.join(' '));
 
     var cp = child_process.spawn(file, args, options);
     if (options.logStream) {