retry apply of platform config

it seems that scaling down addons can fail at times. this can happen
if the kernel is busy swapping things etc. so, we do not block the
platform startup for this operation.

Nov 13 05:20:00 localhost dockerd[28831]: time="2018-11-13T05:20:00.365056059Z" level=error msg="Handler for POST /v1.37/containers/mongodb/update returned error: Cannot update container 6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f: docker-runc did not terminate sucessfully: failed to write 419430400 to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f/memory.memsw.limit_in_bytes: device or resource busy\n: unknown"
Nov 13 05:20:00 localhost node[5486]: box:shell updatemongodb (stderr): Error response from daemon: Cannot update container 6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f: docker-runc did not terminate sucessfully: failed to write 419430400 to memory.memsw.limit_in_bytes: write /sys/fs/cgroup/memory/docker/6532d4a923ce9f10303f2e2aa7f03c35383864f44f3db6abd5c58da3c1a7702f/memory.memsw.limit_in_bytes: device or resource busy
This commit is contained in:
Girish Ramakrishnan
2018-11-12 21:43:53 -08:00
parent 323dfb1853
commit 82037b70e4
3 changed files with 24 additions and 13 deletions

View File

@@ -43,7 +43,6 @@ var accesscontrol = require('./accesscontrol.js'),
rimraf = require('rimraf'),
safe = require('safetydance'),
semver = require('semver'),
settings = require('./settings.js'),
shell = require('./shell.js'),
request = require('request'),
util = require('util');
@@ -355,6 +354,8 @@ function updateAddonConfig(platformConfig, callback) {
mail: Math.max((1 + Math.round(os.totalmem()/(1024*1024*1024)/4)) * 128, 256) * 1024 * 1024
};
debug('updateAddonConfig: %j', platformConfig);
// TODO: this should possibly also rollback memory to default
async.eachSeries([ 'mysql', 'postgresql', 'mail', 'mongodb' ], function iterator(containerName, iteratorCallback) {
const containerConfig = platformConfig[containerName];
@@ -399,16 +400,7 @@ function startAddons(existingInfra, callback) {
debug('startAddons: existing infra. incremental addon create %j', startFuncs.map(function (f) { return f.name; }));
}
async.series(startFuncs, function (error) {
if (error) return callback(error);
// once addons are started/imported, scale them back
settings.getPlatformConfig(function (error, platformConfig) {
if (error) return callback(error);
updateAddonConfig(platformConfig, callback);
});
});
async.series(startFuncs, callback);
}
function getEnvironment(app, callback) {