notifications: better oom message for redis

fixes #795
This commit is contained in:
Girish Ramakrishnan
2021-09-19 17:32:48 -07:00
parent 05b4f96854
commit 50c68cd499
3 changed files with 25 additions and 18 deletions
+10 -7
View File
@@ -99,10 +99,13 @@ async function getContainerInfo(containerId) {
const result = await docker.inspect(containerId);
const appId = safe.query(result, 'Config.Labels.appId', null);
if (appId) return { app: await apps.get(appId) }; // don't get by container id as this can be an exec container
if (!appId) return { app: null, addon: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
return await apps.get(appId); // don't get by container id as this can be an exec container
if (result.Name.startsWith('/redis-')) {
return { app: await apps.get(result.Name.slice('/redis-'.length)), addonName: 'redis' };
} else {
return { addonName: result.Name.slice(1) }; // addon . Name has a '/' in the beginning for some reason
}
}
/*
@@ -123,16 +126,16 @@ async function processDockerEvents(options) {
const containerId = String(event.id);
const [error, info] = await safe(getContainerInfo(containerId));
const program = error ? containerId : (info.app ? info.app.fqdn : info.addon.name);
const program = error ? containerId : (info.addonName || info.app.fqdn);
const now = Date.now();
// do not send mails for dev apps
const notifyUser = !(info.app && info.app.debugMode) && ((now - gLastOomMailTime) > OOM_EVENT_LIMIT);
debug(`OOM ${program} notifyUser: ${notifyUser}. lastOomTime: ${gLastOomMailTime} (now: ${now})`);
// do not send mails for dev apps
if (notifyUser) {
// app can be null for addon containers
await eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addon: info.addon || null, app: info.app || null });
await eventlog.add(eventlog.ACTION_APP_OOM, auditSource.HEALTH_MONITOR, { event, containerId, addonName: info?.addonName || null, app: info?.app || null });
gLastOomMailTime = now;
}
+14 -10
View File
@@ -120,21 +120,25 @@ async function list(filters, page, perPage) {
return results;
}
async function oomEvent(eventId, app, addon, containerId /*, event*/) {
async function oomEvent(eventId, containerId, app, addonName /*, event*/) {
assert.strictEqual(typeof eventId, 'string');
assert.strictEqual(typeof app, 'object');
assert.strictEqual(typeof addon, 'object');
assert.strictEqual(typeof containerId, 'string');
assert.strictEqual(typeof app, 'object');
assert(addonName === null || typeof addonName === 'string');
assert(app || addon);
assert(app || addonName);
let title, message;
if (app) {
title = `The application at ${app.fqdn} ran out of memory.`;
message = `The application has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.dashboardOrigin()}/#/app/${app.id}/resources)`;
} else if (addon) {
title = `The ${addon.name} service ran out of memory`;
if (addonName) {
if (app) {
title = `The ${addonName} service of the app at ${app.fqdn} ran out of memory`;
} else {
title = `The ${addonName} service ran out of memory`;
}
message = `The service has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.dashboardOrigin()}/#/services)`;
} else if (app) {
title = `The app at ${app.fqdn} ran out of memory.`;
message = `The app has been restarted automatically. If you see this notification often, consider increasing the [memory limit](${settings.dashboardOrigin()}/#/app/${app.id}/resources)`;
}
await add(eventId, title, message);
@@ -247,7 +251,7 @@ async function onEvent(id, action, source, data) {
switch (action) {
case eventlog.ACTION_APP_OOM:
return await oomEvent(id, data.app, data.addon, data.containerId, data.event);
return await oomEvent(id, data.containerId, data.app, data.addonName, data.event);
case eventlog.ACTION_APP_UPDATE_FINISH:
if (source.username !== auditSource.CRON.username) return; // updated by user