diff --git a/src/logcollector.js b/src/logcollector.js index 21930c033..f9e5c8ef4 100644 --- a/src/logcollector.js +++ b/src/logcollector.js @@ -12,26 +12,57 @@ var assert = require('assert'), var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh'); +var CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 30; // 30 min +var CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp'; +var CRASH_LOG_STASH_FILE = '/tmp/crashlog'; + function collectLogs(unitName, callback) { assert.strictEqual(typeof unitName, 'string'); assert.strictEqual(typeof callback, 'function'); var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' }); + logs = '\n\n======= ' + unitName + ' =======\n\n' + logs + '\n\n'; + callback(null, logs); } +function stashLogs(logs) { + // append here + safe.fs.writeFileSync(CRASH_LOG_STASH_FILE, logs, { flag: 'a' }); +} + function sendFailureLogs(processName, options) { assert.strictEqual(typeof processName, 'string'); assert.strictEqual(typeof options, 'object'); - collectLogs(options.unit || processName, function (error, result) { + collectLogs(options.unit || processName, function (error, newLogs) { if (error) { console.error('Failed to collect logs.', error); - result = util.format('Failed to collect logs.', error); + newLogs = util.format('Failed to collect logs.', error); } console.log('Sending failure logs for', processName); - mailer.unexpectedExit(processName, result); + var timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8'); + + // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window + if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) { + console.log('Crash log already sent within window. Stashing logs.'); + return stashLogs(newLogs); + } + + var stashedLogs = safe.fs.readFileSync(CRASH_LOG_STASH_FILE, 'utf8'); + var compiledLogs = stashedLogs ? (stashedLogs + newLogs) : newLogs; + + mailer.unexpectedExit(processName, compiledLogs, function (error) { + if (error) { + console.log('Error sending crashlog. Stashing logs.'); + return stashLogs(newLogs); + } + + // write the new timestamp file and delete stash file + safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now())); + safe.fs.unlinkSync(CRASH_LOG_STASH_FILE); + }); }); } diff --git a/src/mailer.js b/src/mailer.js index e546aedc3..8a493d74f 100644 --- a/src/mailer.js +++ b/src/mailer.js @@ -48,6 +48,8 @@ var assert = require('assert'), util = require('util'), _ = require('underscore'); +var NOOP_CALLBACK = function (error) { if (error) console.error(error); }; + var MAIL_TEMPLATES_DIR = path.join(__dirname, 'mail_templates'); var gMailQueue = [ ], @@ -149,14 +151,15 @@ function processQueue() { // note : this function should NOT access the database. it is called by the crashnotifier // which does not initialize mailer or the databse -function sendMails(queue) { +function sendMails(queue, callback) { assert(util.isArray(queue)); + callback = callback || NOOP_CALLBACK; docker.getContainer('mail').inspect(function (error, data) { - if (error) return console.error(error); + if (error) return callback(error); var mailServerIp = safe.query(data, 'NetworkSettings.Networks.cloudron.IPAddress'); - if (!mailServerIp) return debug('Error querying mail server IP'); + if (!mailServerIp) return callback('Error querying mail server IP'); var transport = nodemailer.createTransport(smtpTransport({ host: mailServerIp, @@ -173,6 +176,8 @@ function sendMails(queue) { callback(null); }, function done() { debug('Done processing mail queue'); + + callback(null); }); }); } @@ -474,9 +479,10 @@ function oomEvent(program, context) { // this function bypasses the queue intentionally. it is also expected to work without the mailer module initialized // NOTE: crashnotifier should be able to send mail when there is no db -function unexpectedExit(program, context) { +function unexpectedExit(program, context, callback) { assert.strictEqual(typeof program, 'string'); assert.strictEqual(typeof context, 'string'); + assert.strictEqual(typeof callback, 'function'); var mailOptions = { from: mailConfig().from, @@ -485,7 +491,7 @@ function unexpectedExit(program, context) { text: render('unexpected_exit.ejs', { fqdn: config.fqdn(), program: program, context: context, format: 'text' }) }; - sendMails([ mailOptions ]); + sendMails([ mailOptions ], callback); } function sendFeedback(user, type, subject, description) {