diff --git a/migrations/20190213000125-apps-add-healthTime.js b/migrations/20190213000125-apps-add-healthTime.js new file mode 100644 index 000000000..fb1ce3526 --- /dev/null +++ b/migrations/20190213000125-apps-add-healthTime.js @@ -0,0 +1,17 @@ +'use strict'; + +exports.up = function(db, callback) { + var cmd = 'ALTER TABLE apps ADD COLUMN healthTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP'; + + db.runSql(cmd, function (error) { + if (error) console.error(error); + callback(error); + }); +}; + +exports.down = function(db, callback) { + db.runSql('ALTER TABLE apps DROP COLUMN healthTime', function (error) { + if (error) console.error(error); + callback(error); + }); +}; diff --git a/migrations/schema.sql b/migrations/schema.sql index c18a94ceb..e392fe7e9 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -65,6 +65,7 @@ CREATE TABLE IF NOT EXISTS apps( installationProgress TEXT, runState VARCHAR(512), health VARCHAR(128), + healthTime TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, // when the app last responded containerId VARCHAR(128), manifestJson TEXT, httpPort INTEGER, // this is the nginx proxy port and not manifest.httpPort diff --git a/src/appdb.js b/src/appdb.js index 4877053b8..8b20793bc 100644 --- a/src/appdb.js +++ b/src/appdb.js @@ -70,7 +70,7 @@ var APPS_FIELDS_PREFIXED = [ 'apps.id', 'apps.appStoreId', 'apps.installationSta 'apps.accessRestrictionJson', 'apps.restoreConfigJson', 'apps.oldConfigJson', 'apps.updateConfigJson', 'apps.memoryLimit', 'apps.xFrameOptions', 'apps.sso', 'apps.debugModeJson', 'apps.robotsTxt', 'apps.enableBackup', 'apps.creationTime', 'apps.updateTime', 'apps.ownerId', 'apps.mailboxName', 'apps.enableAutomaticUpdate', - 'apps.dataDir', 'apps.ts' ].join(','); + 'apps.dataDir', 'apps.ts', 'apps.healthTime' ].join(','); var PORT_BINDINGS_FIELDS = [ 'hostPort', 'type', 'environmentVariable', 'appId' ].join(','); @@ -476,12 +476,13 @@ function updateWithConstraints(id, app, constraints, callback) { } // not sure if health should influence runState -function setHealth(appId, health, callback) { +function setHealth(appId, health, healthTime, callback) { assert.strictEqual(typeof appId, 'string'); assert.strictEqual(typeof health, 'string'); + assert(util.isDate(healthTime)); assert.strictEqual(typeof callback, 'function'); - var values = { health: health }; + var values = { health, healthTime }; var constraints = 'AND runState NOT LIKE "pending_%" AND installationState = "installed"'; diff --git a/src/apphealthmonitor.js b/src/apphealthmonitor.js index d2b03d465..50e0f4b30 100644 --- a/src/apphealthmonitor.js +++ b/src/apphealthmonitor.js @@ -17,7 +17,6 @@ exports = module.exports = { const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable const UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes -let gHealthInfo = { }; // { time, appDownEvent } const OOM_MAIL_LIMIT = 60 * 60 * 1000; // 60 minutes let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago @@ -35,35 +34,29 @@ function setHealth(app, health, callback) { assert.strictEqual(typeof health, 'string'); assert.strictEqual(typeof callback, 'function'); - var now = new Date(); - - if (!(app.id in gHealthInfo)) { // add new apps to list - gHealthInfo[app.id] = { time: now, appDownEvent: false }; - } + let now = new Date(), healthTime = app.healthTime, curHealth = app.health; if (health === appdb.HEALTH_HEALTHY) { - gHealthInfo[app.id].time = now; - if (!gHealthInfo[app.id].appDownEvent) return callback(null); + healthTime = now; + if (curHealth === appdb.HEALTH_UNHEALTHY) { + debugApp(app, 'app switched from unhealthy to healthy'); - // do not send mails for dev apps - if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, AUDIT_SOURCE, { app: app }); + // do not send mails for dev apps + if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, AUDIT_SOURCE, { app: app }); + } + } else if (Math.abs(now - healthTime) > UNHEALTHY_THRESHOLD) { + if (curHealth === appdb.HEALTH_HEALTHY) { + debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000)); - gHealthInfo[app.id].appDownEvent = false; - } else if (Math.abs(now - gHealthInfo[app.id].time) > UNHEALTHY_THRESHOLD) { - if (gHealthInfo[app.id].appDownEvent) return callback(null); - - debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000)); - - // do not send mails for dev apps - if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, AUDIT_SOURCE, { app: app }); - - gHealthInfo[app.id].appDownEvent = true; + // do not send mails for dev apps + if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, AUDIT_SOURCE, { app: app }); + } } else { - debugApp(app, 'waiting for %s seconds to update the app health', (Math.abs(now - gHealthInfo[app.id].time) - UNHEALTHY_THRESHOLD)/1000); + debugApp(app, 'waiting for %s seconds to update the app health', (UNHEALTHY_THRESHOLD - Math.abs(now - healthTime))/1000); return callback(null); } - appdb.setHealth(app.id, health, function (error) { + appdb.setHealth(app.id, health, healthTime, function (error) { if (error && error.reason === DatabaseError.NOT_FOUND) return callback(null); // app uninstalled? if (error) return callback(error); diff --git a/src/test/database-test.js b/src/test/database-test.js index 0ac060118..acddcfe18 100644 --- a/src/test/database-test.js +++ b/src/test/database-test.js @@ -1124,7 +1124,7 @@ describe('database', function () { appdb.get(APP_0.id, function (error, result) { expect(error).to.be(null); expect(result).to.be.an('object'); - expect(_.omit(result, ['creationTime', 'updateTime', 'ts'])).to.be.eql(APP_0); + expect(_.omit(result, ['creationTime', 'updateTime', 'ts', 'healthTime'])).to.be.eql(APP_0); done(); }); }); @@ -1162,7 +1162,7 @@ describe('database', function () { appdb.get(APP_0.id, function (error, result) { expect(error).to.be(null); expect(result).to.be.an('object'); - expect(_.omit(result, ['creationTime', 'updateTime', 'ts'])).to.be.eql(APP_0); + expect(_.omit(result, ['creationTime', 'updateTime', 'ts', 'healthTime'])).to.be.eql(APP_0); done(); }); }); @@ -1172,7 +1172,7 @@ describe('database', function () { appdb.getByHttpPort(APP_0.httpPort, function (error, result) { expect(error).to.be(null); expect(result).to.be.an('object'); - expect(_.omit(result, ['creationTime', 'updateTime', 'ts'])).to.be.eql(APP_0); + expect(_.omit(result, ['creationTime', 'updateTime', 'ts', 'healthTime'])).to.be.eql(APP_0); done(); }); }); @@ -1197,8 +1197,8 @@ describe('database', function () { expect(error).to.be(null); expect(result).to.be.an(Array); expect(result.length).to.be(2); - expect(_.omit(result[0], ['creationTime', 'updateTime','ts'])).to.be.eql(APP_0); - expect(_.omit(result[1], ['creationTime', 'updateTime','ts'])).to.be.eql(APP_1); + expect(_.omit(result[0], ['creationTime', 'updateTime','ts', 'healthTime'])).to.be.eql(APP_0); + expect(_.omit(result[1], ['creationTime', 'updateTime','ts', 'healthTime'])).to.be.eql(APP_1); done(); }); }); @@ -1239,7 +1239,7 @@ describe('database', function () { }); it('cannot set app as healthy because app is not installed', function (done) { - appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, function (error) { + appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, new Date(), function (error) { expect(error).to.be.ok(); done(); }); @@ -1249,7 +1249,7 @@ describe('database', function () { appdb.update(APP_1.id, { runState: appdb.RSTATE_PENDING_STOP, installationState: appdb.ISTATE_INSTALLED }, function (error) { expect(error).to.be(null); - appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, function (error) { + appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, new Date(), function (error) { expect(error).to.be.ok(); done(); }); @@ -1260,7 +1260,7 @@ describe('database', function () { appdb.update(APP_1.id, { runState: null, installationState: appdb.ISTATE_INSTALLED }, function (error) { expect(error).to.be(null); - appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, function (error) { + appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, new Date(), function (error) { expect(error).to.be.ok(); done(); }); @@ -1271,7 +1271,7 @@ describe('database', function () { appdb.update(APP_1.id, { runState: appdb.RSTATE_RUNNING, installationState: appdb.ISTATE_INSTALLED }, function (error) { expect(error).to.be(null); - appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, function (error) { + appdb.setHealth(APP_1.id, appdb.HEALTH_HEALTHY, new Date(), function (error) { expect(error).to.be(null); appdb.get(APP_1.id, function (error, app) { expect(error).to.be(null); @@ -1283,7 +1283,7 @@ describe('database', function () { }); it('cannot set health of unknown app', function (done) { - appdb.setHealth('randomId', appdb.HEALTH_HEALTHY, function (error) { + appdb.setHealth('randomId', appdb.HEALTH_HEALTHY, new Date(), function (error) { expect(error).to.be.ok(); done(); });