diff --git a/CHANGES b/CHANGES index 40c7eca89..eede7bb7e 100644 --- a/CHANGES +++ b/CHANGES @@ -1530,4 +1530,5 @@ * Mailbox and lists UI is now always visible (but disabled) when incoming email is disabled * Fix issue where long passwords were not accepted * DNS and backup credential secrets are not returned in API calls anymore +* Send notification when an app that went down, came back up diff --git a/src/apphealthmonitor.js b/src/apphealthmonitor.js index 7ab762968..7eab185b9 100644 --- a/src/apphealthmonitor.js +++ b/src/apphealthmonitor.js @@ -17,7 +17,7 @@ exports = module.exports = { const HEALTHCHECK_INTERVAL = 10 * 1000; // every 10 seconds. this needs to be small since the UI makes only healthy apps clickable const UNHEALTHY_THRESHOLD = 10 * 60 * 1000; // 10 minutes -let gHealthInfo = { }; // { time, emailSent } +let gHealthInfo = { }; // { time, appDownEvent } const OOM_MAIL_LIMIT = 60 * 60 * 1000; // 60 minutes let gLastOomMailTime = Date.now() - (5 * 60 * 1000); // pretend we sent email 5 minutes ago @@ -36,22 +36,28 @@ function setHealth(app, health, callback) { var now = new Date(); if (!(app.id in gHealthInfo)) { // add new apps to list - gHealthInfo[app.id] = { time: now, emailSent: false }; + gHealthInfo[app.id] = { time: now, appDownEvent: false }; } if (health === appdb.HEALTH_HEALTHY) { gHealthInfo[app.id].time = now; + if (!gHealthInfo[app.id].appDownEvent) return callback(null); + + // do not send mails for dev apps + if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_UP, { app: app }, {}); + + gHealthInfo[app.id].appDownEvent = false; } else if (Math.abs(now - gHealthInfo[app.id].time) > UNHEALTHY_THRESHOLD) { - if (gHealthInfo[app.id].emailSent) return callback(null); + if (gHealthInfo[app.id].appDownEvent) return callback(null); debugApp(app, 'marking as unhealthy since not seen for more than %s minutes', UNHEALTHY_THRESHOLD/(60 * 1000)); // do not send mails for dev apps if (!app.debugMode) eventlog.add(eventlog.ACTION_APP_DOWN, { app: app }, {}); - gHealthInfo[app.id].emailSent = true; + gHealthInfo[app.id].appDownEvent = true; } else { - debugApp(app, 'waiting for sometime to update the app health'); + debugApp(app, 'waiting for %s seconds to update the app health', (Math.abs(now - gHealthInfo[app.id].time) - UNHEALTHY_THRESHOLD)/1000); return callback(null); } diff --git a/src/eventlog.js b/src/eventlog.js index 19d4193b7..0a6891aaa 100644 --- a/src/eventlog.js +++ b/src/eventlog.js @@ -19,6 +19,7 @@ exports = module.exports = { ACTION_APP_UPDATE: 'app.update', ACTION_APP_LOGIN: 'app.login', ACTION_APP_OOM: 'app.oom', + ACTION_APP_UP: 'app.up', ACTION_APP_DOWN: 'app.down', ACTION_APP_TASK_CRASH: 'app.task.crash', @@ -115,6 +116,8 @@ function add(action, source, data, callback) { notifications.oomEvent(id, source.app ? source.app.id : source.containerId, { app: source.app, details: data }); } else if (action === exports.ACTION_APP_DOWN) { notifications.appDied(id, source.app); + } else if (action === exports.ACTION_APP_UP) { + notifications.appUp(id, source.app); } else if (action === exports.ACTION_APP_TASK_CRASH) { notifications.apptaskCrash(id, source.appId, data.crashLogFile); } else if (action === exports.ACTION_PROCESS_CRASH) { diff --git a/src/mail_templates/app_up.ejs b/src/mail_templates/app_up.ejs new file mode 100644 index 000000000..9d842ab58 --- /dev/null +++ b/src/mail_templates/app_up.ejs @@ -0,0 +1,14 @@ +<%if (format === 'text') { %> + +Dear Cloudron Admin, + +The application '<%= title %>' installed at <%= appFqdn %> is back online +and responding to health checks. + +Powered by https://cloudron.io + +Sent at: <%= new Date().toUTCString() %> + +<% } else { %> + +<% } %> diff --git a/src/mailer.js b/src/mailer.js index 7db8136dc..566a068a8 100644 --- a/src/mailer.js +++ b/src/mailer.js @@ -12,6 +12,7 @@ exports = module.exports = { sendInvite: sendInvite, unexpectedExit: unexpectedExit, + appUp: appUp, appDied: appDied, oomEvent: oomEvent, @@ -311,6 +312,26 @@ function passwordReset(user) { }); } +function appUp(mailTo, app) { + assert.strictEqual(typeof mailTo, 'string'); + assert.strictEqual(typeof app, 'object'); + + debug('Sending mail for app %s @ %s up', app.id, app.fqdn); + + getMailConfig(function (error, mailConfig) { + if (error) return debug('Error getting mail details:', error); + + var mailOptions = { + from: mailConfig.notificationFrom, + to: mailTo, + subject: util.format('[%s] App %s is back online', mailConfig.cloudronName, app.fqdn), + text: render('app_up.ejs', { title: app.manifest.title, appFqdn: app.fqdn, format: 'text' }) + }; + + enqueue(mailOptions); + }); +} + function appDied(mailTo, app) { assert.strictEqual(typeof mailTo, 'string'); assert.strictEqual(typeof app, 'object'); diff --git a/src/notifications.js b/src/notifications.js index ac961907f..9519d24f8 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -14,6 +14,7 @@ exports = module.exports = { userRemoved: userRemoved, adminChanged: adminChanged, oomEvent: oomEvent, + appUp: appUp, appDied: appDied, processCrash: processCrash, apptaskCrash: apptaskCrash, @@ -219,6 +220,21 @@ function oomEvent(eventId, program, context) { }); } +function appUp(eventId, app) { + assert.strictEqual(typeof eventId, 'string'); + assert.strictEqual(typeof app, 'object'); + + // also send us a notification mail + if (config.provider() === 'caas') mailer.appDied('support@cloudron.io', app); + + actionForAllAdmins([], function (admin, callback) { + mailer.appUp(admin.email, app); + add(admin.id, eventId, `App ${app.fqdn} is back online`, `The application ${app.manifest.title} installed at ${app.fqdn} is back online.`, '/#/apps', callback); + }, function (error) { + if (error) console.error(error); + }); +} + function appDied(eventId, app) { assert.strictEqual(typeof eventId, 'string'); assert.strictEqual(typeof app, 'object');