send logs when apptask fails

This commit is contained in:
Girish Ramakrishnan
2016-04-19 18:39:44 -07:00
parent 15653cb3f8
commit eadff099eb
5 changed files with 21 additions and 15 deletions
+2 -2
View File
@@ -2,7 +2,7 @@
'use strict';
var sendCrashNotification = require('./src/logcollector').sendCrashNotification;
var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
function main() {
if (process.argv.length !== 3) return console.error('Usage: crashnotifier.js <processName>');
@@ -10,7 +10,7 @@ function main() {
var processName = process.argv[2];
console.log('Started crash notifier for', processName);
sendCrashNotification(processName);
sendFailureLogs(processName, { unit: processName });
}
main();
+11 -7
View File
@@ -1,7 +1,7 @@
'use strict';
exports = module.exports = {
sendCrashNotification: sendCrashNotification
sendFailureLogs: sendFailureLogs
};
var assert = require('assert'),
@@ -12,22 +12,26 @@ var assert = require('assert'),
var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
function collectLogs(program, callback) {
assert.strictEqual(typeof program, 'string');
function collectLogs(unitName, callback) {
assert.strictEqual(typeof unitName, 'string');
assert.strictEqual(typeof callback, 'function');
var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + program, { encoding: 'utf8' });
var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
callback(null, logs);
}
function sendCrashNotification(processName) {
function sendFailureLogs(processName, options) {
assert.strictEqual(typeof processName, 'string');
assert.strictEqual(typeof options, 'object');
collectLogs(processName, function (error, result) {
if (error) {
console.error('Failed to collect logs.', error);
result = util.format('Failed to collect logs.', error);
}
console.log('Sending crash notification email for', processName);
mailer.sendFailureLogs(processName, result);
console.log('Sending failure logs for', processName);
mailer.unexpectedExit(options.unit || processName, result);
});
}
@@ -2,7 +2,7 @@
Dear Cloudron Team,
Unfortunately <%= program %> on <%= fqdn %> failed unexpectedly!
Unfortunately <%= program %> on <%= fqdn %> exited unexpectedly!
Please see some excerpt of the logs below.
+3 -3
View File
@@ -12,7 +12,7 @@ exports = module.exports = {
appUpdateAvailable: appUpdateAvailable,
sendInvite: sendInvite,
sendFailureLogs: sendFailureLogs,
unexpectedExit: unexpectedExit,
appDied: appDied,
@@ -395,7 +395,7 @@ function certificateRenewed(domain, message) {
// this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized
// crashnotifier should be able to send mail when there is no db
function sendFailureLogs(program, context) {
function unexpectedExit(program, context) {
assert.strictEqual(typeof program, 'string');
assert.strictEqual(typeof context, 'string');
@@ -403,7 +403,7 @@ function sendFailureLogs(program, context) {
from: config.adminEmail(),
to: 'admin@cloudron.io',
subject: util.format('[%s] %s exited unexpectedly', config.fqdn(), program),
text: render('failure_notification.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' })
};
sendMails([ mailOptions ]);
+4 -2
View File
@@ -19,7 +19,7 @@ var appdb = require('./appdb.js'),
cloudron = require('./cloudron.js'),
debug = require('debug')('box:taskmanager'),
locker = require('./locker.js'),
sendCrashNotification = require('./logcollector.js').sendCrashNotification,
sendFailureLogs = require('./logcollector.js').sendFailureLogs,
util = require('util'),
_ = require('underscore');
@@ -143,8 +143,10 @@ function startAppTask(appId, callback) {
debug('Task for %s pid %s completed with status %s', appId, pid, code);
if (code === null /* signal */ || (code !== 0 && code !== 50)) { // apptask crashed
debug('Apptask crashed with code %s and signal %s', code, signal);
sendCrashNotification('box');
sendFailureLogs('apptask', { unit: 'box' });
appdb.update(appId, { installationState: appdb.ISTATE_ERROR, installationProgress: 'Apptask crashed with code ' + code + ' and signal ' + signal }, NOOP_CALLBACK);
} else if (code === 50) {
sendFailureLogs('apptask', { unit: 'box' });
}
delete gActiveTasks[appId];
locker.unlock(locker.OP_APPTASK); // unlock event will trigger next task