diff --git a/crashnotifierservice.js b/crashnotifierservice.js index 93bc9edb3..38085c9d2 100755 --- a/crashnotifierservice.js +++ b/crashnotifierservice.js @@ -4,7 +4,7 @@ var database = require('./src/database.js'); -var sendFailureLogs = require('./src/logcollector').sendFailureLogs; +var crashNotifier = require('./src/crashnotifier.js'); // This is triggered by systemd with the crashed unit name as argument function main() { @@ -17,7 +17,11 @@ function main() { database.initialize(function (error) { if (error) return console.error('Cannot connect to database. Unable to send crash log.', error); - sendFailureLogs(unitName); + crashNotifier.sendFailureLogs(unitName, function (error) { + if (error) console.error(error); + + process.exit(); + }); }); } diff --git a/setup/start.sh b/setup/start.sh index 8f72df880..00905546e 100755 --- a/setup/start.sh +++ b/setup/start.sh @@ -60,7 +60,10 @@ mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d" mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d" mkdir -p "${PLATFORM_DATA_DIR}/acme" mkdir -p "${PLATFORM_DATA_DIR}/backup" -mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" "${PLATFORM_DATA_DIR}/logs/updater" "${PLATFORM_DATA_DIR}/logs/tasks" +mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \ + "${PLATFORM_DATA_DIR}/logs/updater" \ + "${PLATFORM_DATA_DIR}/logs/tasks" \ + "${PLATFORM_DATA_DIR}/logs/crash" mkdir -p "${PLATFORM_DATA_DIR}/update" mkdir -p "${BOX_DATA_DIR}/appicons" @@ -135,8 +138,9 @@ echo "==> Configuring logrotate" if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf fi -cp "${script_dir}/start/box-logrotate" "${script_dir}/start/app-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/" -chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/box-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/app-logrotate" +cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/" +rm -f "${PLATFORM_DATA_DIR}/logrotate.d/box-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/app-logrotate" # remove pre 3.6 config files +chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/" echo "==> Adding motd message for admins" cp "${script_dir}/start/cloudron-motd" /etc/update-motd.d/92-cloudron diff --git a/setup/start/app-logrotate b/setup/start/logrotate/app similarity index 85% rename from setup/start/app-logrotate rename to setup/start/logrotate/app index ce765d34e..f83e7f5eb 100644 --- a/setup/start/app-logrotate +++ b/setup/start/logrotate/app @@ -1,4 +1,4 @@ -# logrotate config for app logs +# logrotate config for app and crash logs /home/yellowtent/platformdata/logs/*/*.log { # only keep one rotated file, we currently do not send that over the api diff --git a/setup/start/box-logrotate b/setup/start/logrotate/box similarity index 100% rename from setup/start/box-logrotate rename to setup/start/logrotate/box diff --git a/src/crashnotifier.js b/src/crashnotifier.js new file mode 100644 index 000000000..b8b1b9ab4 --- /dev/null +++ b/src/crashnotifier.js @@ -0,0 +1,61 @@ +'use strict'; + +exports = module.exports = { + sendFailureLogs: sendFailureLogs +}; + +var assert = require('assert'), + eventlog = require('./eventlog.js'), + safe = require('safetydance'), + path = require('path'), + paths = require('./paths.js'), + util = require('util'); + +const COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh'); + +const CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min +const CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp'; + +const AUDIT_SOURCE = { userId: null, username: 'healthmonitor' }; + +function collectLogs(unitName, callback) { + assert.strictEqual(typeof unitName, 'string'); + assert.strictEqual(typeof callback, 'function'); + + var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' }); + if (!logs) return callback(safe.error); + + callback(null, logs); +} + +function sendFailureLogs(unitName, callback) { + assert.strictEqual(typeof unitName, 'string'); + assert.strictEqual(typeof callback, 'function'); + + // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window + const timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8'); + if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) { + console.log('Crash log already sent within window'); + return callback(); + } + + collectLogs(unitName, function (error, logs) { + if (error) { + console.error('Failed to collect logs.', error); + logs = util.format('Failed to collect logs.', error); + } + + const crashLogFile = `${new Date().toISOString()}.log`; + console.log(`Sending failure logs for ${unitName} at ${crashLogFile}`); + + if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, crashLogFile), logs)) console.log(`Failed to stash logs to ${crashLogFile}:`, safe.error); + + eventlog.add(eventlog.ACTION_PROCESS_CRASH, AUDIT_SOURCE, { processName: unitName, crashLogFile: crashLogFile }, function (error) { + if (error) console.log(`Error sending crashlog. Logs stashed at ${crashLogFile}`); + + safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now())); + + callback(); + }); + }); +} diff --git a/src/logcollector.js b/src/logcollector.js deleted file mode 100644 index 6bb2fbf3d..000000000 --- a/src/logcollector.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -exports = module.exports = { - sendFailureLogs: sendFailureLogs -}; - -var assert = require('assert'), - eventlog = require('./eventlog.js'), - safe = require('safetydance'), - path = require('path'), - paths = require('./paths.js'), - util = require('util'); - -var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh'); - -var CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min -var CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp'; -var CRASH_LOG_STASH_FILE = '/tmp/crashlog'; - -const AUDIT_SOURCE = { userId: null, username: 'healthmonitor' }; - -function collectLogs(unitName, callback) { - assert.strictEqual(typeof unitName, 'string'); - assert.strictEqual(typeof callback, 'function'); - - var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' }); - if (!logs) return callback(safe.error); - - logs = logs + '\n\n=====================================\n\n'; - - // special case for box since the real logs are at path.join(paths.LOG_DIR, 'box.log') - if (unitName === 'box.service') { - logs += safe.child_process.execSync('tail --lines=500 ' + path.join(paths.LOG_DIR, 'box.log'), { encoding: 'utf8' }); - } - - callback(null, logs); -} - -function sendFailureLogs(unitName) { - assert.strictEqual(typeof unitName, 'string'); - - collectLogs(unitName, function (error, logs) { - if (error) { - console.error('Failed to collect logs.', error); - logs = util.format('Failed to collect logs.', error); - } - - console.log('Sending failure logs for', unitName); - - if (!safe.fs.writeFileSync(CRASH_LOG_STASH_FILE, logs)) console.log(`Failed to stash logs to ${CRASH_LOG_STASH_FILE}`); - - var timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8'); - - // check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window - if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) { - console.log('Crash log already sent within window. Stashing logs.'); - return; - } - - eventlog.add(eventlog.ACTION_PROCESS_CRASH, AUDIT_SOURCE, { processName: unitName, crashLogFile: CRASH_LOG_STASH_FILE }, function (error) { - if (error) console.log(`Error sending crashlog. Logs stashed at ${CRASH_LOG_STASH_FILE}`); - - // write the new timestamp file and delete stash file - safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now())); - }); - }); -} diff --git a/src/notifications.js b/src/notifications.js index 0d15110db..7696f692b 100644 --- a/src/notifications.js +++ b/src/notifications.js @@ -26,6 +26,8 @@ let assert = require('assert'), eventlog = require('./eventlog.js'), mailer = require('./mailer.js'), notificationdb = require('./notificationdb.js'), + path = require('path'), + paths = require('./paths.js'), safe = require('safetydance'), users = require('./users.js'), util = require('util'); @@ -234,14 +236,14 @@ function processCrash(eventId, processName, crashLogFile) { assert.strictEqual(typeof crashLogFile, 'string'); var subject = `${processName} exited unexpectedly`; - var crashLogs = safe.fs.readFileSync(crashLogFile, 'utf8') || `No logs found at ${crashLogFile}`; + var crashLogs = safe.fs.readFileSync(path.join(paths.CRASH_LOG_DIR, crashLogFile), 'utf8') || `No logs found at ${crashLogFile}`; // also send us a notification mail if (config.provider() === 'caas') mailer.unexpectedExit('support@cloudron.io', subject, crashLogs); actionForAllAdmins([], function (admin, callback) { mailer.unexpectedExit(admin.email, subject, crashLogs); - add(admin.id, eventId, subject, 'Detailed logs have been sent to your email address.', callback); + add(admin.id, eventId, subject, `The service has been restarted automatically. Crash logs are available [here](/logs.html?crash=${crashLogFile})`, callback); }, function (error) { if (error) console.error(error); }); diff --git a/src/paths.js b/src/paths.js index 5d588adb3..6384ddf31 100644 --- a/src/paths.js +++ b/src/paths.js @@ -36,6 +36,7 @@ exports = module.exports = { LOG_DIR: path.join(config.baseDir(), 'platformdata/logs'), TASKS_LOG_DIR: path.join(config.baseDir(), 'platformdata/logs/tasks'), + CRASH_LOG_DIR: path.join(config.baseDir(), 'platformdata/logs/crash'), // this pattern is for the cloudron logs API route to work BACKUP_LOG_FILE: path.join(config.baseDir(), 'platformdata/logs/backup/app.log'), diff --git a/src/scripts/collectlogs.sh b/src/scripts/collectlogs.sh index 64238202b..c76f4b842 100755 --- a/src/scripts/collectlogs.sh +++ b/src/scripts/collectlogs.sh @@ -41,3 +41,10 @@ docker ps echo echo docker network inspect cloudron +echo +echo +echo "box" +echo "---" +tail --lines=500 /home/yellowtent/platformdata/logs/box.log + +