diff --git a/CHANGES b/CHANGES index 559dc523b..511712ef8 100644 --- a/CHANGES +++ b/CHANGES @@ -1219,4 +1219,6 @@ * DNS records can now be a A record or a CNAME record * Fix generation of fallback certificates to include naked domain * Merge multi-string DKIM records +* scheduler: do not start cron jobs all at once +* scheduler: give cron jobs a grace period of 30 minutes to complete diff --git a/src/docker.js b/src/docker.js index c95c88e1a..f7ac6da7b 100644 --- a/src/docker.js +++ b/src/docker.js @@ -15,6 +15,7 @@ exports = module.exports = { createSubcontainer: createSubcontainer, getContainerIdByIp: getContainerIdByIp, inspect: inspect, + inspectByName: inspect, execContainer: execContainer }; diff --git a/src/scheduler.js b/src/scheduler.js index b0ef06c15..73bbddc78 100644 --- a/src/scheduler.js +++ b/src/scheduler.js @@ -75,8 +75,6 @@ function sync(callback) { } function killContainer(containerName, callback) { - if (!containerName) return callback(); - async.series([ docker.stopContainerByName.bind(null, containerName), docker.deleteContainerByName.bind(null, containerName) @@ -140,6 +138,8 @@ function doTask(appId, taskName, callback) { assert.strictEqual(typeof taskName, 'string'); assert(!callback || typeof callback === 'function'); + const JOB_MAX_TIME = 30 * 60 * 1000; // 30 minutes + callback = callback || NOOP_CALLBACK; debug('Executing task %s/%s', appId, taskName); @@ -154,16 +154,26 @@ function doTask(appId, taskName, callback) { var containerName = app.id + '-' + taskName; - killContainer(containerName, function (error) { - if (error) return callback(error); + docker.inspectByName(containerName, function (err, data) { + if (!err && data && data.State.Running === true) { + const jobStartTime = gState[appId].cronJobs[taskName].lastDate(); + if (new Date() - jobStartTime < JOB_MAX_TIME) { + debug('task %s of app %s skipped since it is within the job max time %s', taskName. app.id, jobStartTime); + return callback(); + } + } - debug('Creating subcontainer for %s/%s : %s', app.id, taskName, gState[appId].schedulerConfig[taskName].command); - - // NOTE: if you change container name here, fix addons.js to return correct container names - docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', gState[appId].schedulerConfig[taskName].command ], { } /* options */, function (error, container) { + killContainer(containerName, function (error) { if (error) return callback(error); - docker.startContainer(container.id, callback); + debug('Creating subcontainer for %s/%s : %s', app.id, taskName, gState[appId].schedulerConfig[taskName].command); + + // NOTE: if you change container name here, fix addons.js to return correct container names + docker.createSubcontainer(app, containerName, [ '/bin/sh', '-c', gState[appId].schedulerConfig[taskName].command ], { } /* options */, function (error, container) { + if (error) return callback(error); + + docker.startContainer(container.id, callback); + }); }); }); });