jjkiers/cloudron-box
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Stash logs in crash log directory

Browse Source
This commit is contained in:
Girish Ramakrishnan
2019-03-01 14:40:28 -08:00
parent c361ab954d
commit 7983ff5db2
9 changed files with 87 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crashnotifierservice.js
+6 -2
View File
@@ -4,7 +4,7 @@
var database = require('./src/database.js');
var sendFailureLogs = require('./src/logcollector').sendFailureLogs;
var crashNotifier = require('./src/crashnotifier.js');
// This is triggered by systemd with the crashed unit name as argument
function main() {
@@ -17,7 +17,11 @@ function main() {
database.initialize(function (error) {
if (error) return console.error('Cannot connect to database. Unable to send crash log.', error);
sendFailureLogs(unitName);
crashNotifier.sendFailureLogs(unitName, function (error) {
if (error) console.error(error);
process.exit();
});
});
}
setup/start.sh
+7 -3
View File
@@ -60,7 +60,10 @@ mkdir -p "${PLATFORM_DATA_DIR}/collectd/collectd.conf.d"
mkdir -p "${PLATFORM_DATA_DIR}/logrotate.d"
mkdir -p "${PLATFORM_DATA_DIR}/acme"
mkdir -p "${PLATFORM_DATA_DIR}/backup"
mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" "${PLATFORM_DATA_DIR}/logs/updater" "${PLATFORM_DATA_DIR}/logs/tasks"
mkdir -p "${PLATFORM_DATA_DIR}/logs/backup" \
"${PLATFORM_DATA_DIR}/logs/updater" \
"${PLATFORM_DATA_DIR}/logs/tasks" \
"${PLATFORM_DATA_DIR}/logs/crash"
mkdir -p "${PLATFORM_DATA_DIR}/update"
mkdir -p "${BOX_DATA_DIR}/appicons"
@@ -135,8 +138,9 @@ echo "==> Configuring logrotate"
if ! grep -q "^include ${PLATFORM_DATA_DIR}/logrotate.d" /etc/logrotate.conf; then
echo -e "\ninclude ${PLATFORM_DATA_DIR}/logrotate.d\n" >> /etc/logrotate.conf
fi
cp "${script_dir}/start/box-logrotate" "${script_dir}/start/app-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/"
chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/box-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/app-logrotate"
cp "${script_dir}/start/logrotate/"* "${PLATFORM_DATA_DIR}/logrotate.d/"
rm -f "${PLATFORM_DATA_DIR}/logrotate.d/box-logrotate" "${PLATFORM_DATA_DIR}/logrotate.d/app-logrotate" # remove pre 3.6 config files
chown root:root "${PLATFORM_DATA_DIR}/logrotate.d/"
echo "==> Adding motd message for admins"
cp "${script_dir}/start/cloudron-motd" /etc/update-motd.d/92-cloudron
setup/start/app-logrotate → setup/start/logrotate/app
+1 -1
View File
@@ -1,4 +1,4 @@
# logrotate config for app logs
# logrotate config for app and crash logs
/home/yellowtent/platformdata/logs/*/*.log {
# only keep one rotated file, we currently do not send that over the api
setup/start/box-logrotate → setup/start/logrotate/box
View File
src/crashnotifier.js
+61
View File
@@ -0,0 +1,61 @@
'use strict';
exports = module.exports = {
sendFailureLogs: sendFailureLogs
};
var assert = require('assert'),
eventlog = require('./eventlog.js'),
safe = require('safetydance'),
path = require('path'),
paths = require('./paths.js'),
util = require('util');
const COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
const CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min
const CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp';
const AUDIT_SOURCE = { userId: null, username: 'healthmonitor' };
function collectLogs(unitName, callback) {
assert.strictEqual(typeof unitName, 'string');
assert.strictEqual(typeof callback, 'function');
var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
if (!logs) return callback(safe.error);
callback(null, logs);
}
function sendFailureLogs(unitName, callback) {
assert.strictEqual(typeof unitName, 'string');
assert.strictEqual(typeof callback, 'function');
// check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window
const timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8');
if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) {
console.log('Crash log already sent within window');
return callback();
}
collectLogs(unitName, function (error, logs) {
if (error) {
console.error('Failed to collect logs.', error);
logs = util.format('Failed to collect logs.', error);
}
const crashLogFile = `${new Date().toISOString()}.log`;
console.log(`Sending failure logs for ${unitName} at ${crashLogFile}`);
if (!safe.fs.writeFileSync(path.join(paths.CRASH_LOG_DIR, crashLogFile), logs)) console.log(`Failed to stash logs to ${crashLogFile}:`, safe.error);
eventlog.add(eventlog.ACTION_PROCESS_CRASH, AUDIT_SOURCE, { processName: unitName, crashLogFile: crashLogFile }, function (error) {
if (error) console.log(`Error sending crashlog. Logs stashed at ${crashLogFile}`);
safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
callback();
});
});
}
src/logcollector.js
-67
View File
@@ -1,67 +0,0 @@
'use strict';
exports = module.exports = {
sendFailureLogs: sendFailureLogs
};
var assert = require('assert'),
eventlog = require('./eventlog.js'),
safe = require('safetydance'),
path = require('path'),
paths = require('./paths.js'),
util = require('util');
var COLLECT_LOGS_CMD = path.join(__dirname, 'scripts/collectlogs.sh');
var CRASH_LOG_TIMESTAMP_OFFSET = 1000 * 60 * 60; // 60 min
var CRASH_LOG_TIMESTAMP_FILE = '/tmp/crashlog.timestamp';
var CRASH_LOG_STASH_FILE = '/tmp/crashlog';
const AUDIT_SOURCE = { userId: null, username: 'healthmonitor' };
function collectLogs(unitName, callback) {
assert.strictEqual(typeof unitName, 'string');
assert.strictEqual(typeof callback, 'function');
var logs = safe.child_process.execSync('sudo ' + COLLECT_LOGS_CMD + ' ' + unitName, { encoding: 'utf8' });
if (!logs) return callback(safe.error);
logs = logs + '\n\n=====================================\n\n';
// special case for box since the real logs are at path.join(paths.LOG_DIR, 'box.log')
if (unitName === 'box.service') {
logs += safe.child_process.execSync('tail --lines=500 ' + path.join(paths.LOG_DIR, 'box.log'), { encoding: 'utf8' });
}
callback(null, logs);
}
function sendFailureLogs(unitName) {
assert.strictEqual(typeof unitName, 'string');
collectLogs(unitName, function (error, logs) {
if (error) {
console.error('Failed to collect logs.', error);
logs = util.format('Failed to collect logs.', error);
}
console.log('Sending failure logs for', unitName);
if (!safe.fs.writeFileSync(CRASH_LOG_STASH_FILE, logs)) console.log(`Failed to stash logs to ${CRASH_LOG_STASH_FILE}`);
var timestamp = safe.fs.readFileSync(CRASH_LOG_TIMESTAMP_FILE, 'utf8');
// check if we already sent a mail in the last CRASH_LOG_TIME_OFFSET window
if (timestamp && (parseInt(timestamp) + CRASH_LOG_TIMESTAMP_OFFSET) > Date.now()) {
console.log('Crash log already sent within window. Stashing logs.');
return;
}
eventlog.add(eventlog.ACTION_PROCESS_CRASH, AUDIT_SOURCE, { processName: unitName, crashLogFile: CRASH_LOG_STASH_FILE }, function (error) {
if (error) console.log(`Error sending crashlog. Logs stashed at ${CRASH_LOG_STASH_FILE}`);
// write the new timestamp file and delete stash file
safe.fs.writeFileSync(CRASH_LOG_TIMESTAMP_FILE, String(Date.now()));
});
});
}
src/notifications.js
+4 -2
View File
@@ -26,6 +26,8 @@ let assert = require('assert'),
eventlog = require('./eventlog.js'),
mailer = require('./mailer.js'),
notificationdb = require('./notificationdb.js'),
path = require('path'),
paths = require('./paths.js'),
safe = require('safetydance'),
users = require('./users.js'),
util = require('util');
@@ -234,14 +236,14 @@ function processCrash(eventId, processName, crashLogFile) {
assert.strictEqual(typeof crashLogFile, 'string');
var subject = `${processName} exited unexpectedly`;
var crashLogs = safe.fs.readFileSync(crashLogFile, 'utf8') || `No logs found at ${crashLogFile}`;
var crashLogs = safe.fs.readFileSync(path.join(paths.CRASH_LOG_DIR, crashLogFile), 'utf8') || `No logs found at ${crashLogFile}`;
// also send us a notification mail
if (config.provider() === 'caas') mailer.unexpectedExit('support@cloudron.io', subject, crashLogs);
actionForAllAdmins([], function (admin, callback) {
mailer.unexpectedExit(admin.email, subject, crashLogs);
add(admin.id, eventId, subject, 'Detailed logs have been sent to your email address.', callback);
add(admin.id, eventId, subject, `The service has been restarted automatically. Crash logs are available [here](/logs.html?crash=${crashLogFile})`, callback);
}, function (error) {
if (error) console.error(error);
});
src/paths.js
+1
View File
@@ -36,6 +36,7 @@ exports = module.exports = {
LOG_DIR: path.join(config.baseDir(), 'platformdata/logs'),
TASKS_LOG_DIR: path.join(config.baseDir(), 'platformdata/logs/tasks'),
CRASH_LOG_DIR: path.join(config.baseDir(), 'platformdata/logs/crash'),
// this pattern is for the cloudron logs API route to work
BACKUP_LOG_FILE: path.join(config.baseDir(), 'platformdata/logs/backup/app.log'),
src/scripts/collectlogs.sh
+7
View File
@@ -41,3 +41,10 @@ docker ps
echo
echo
docker network inspect cloudron
echo
echo
echo "box"
echo "---"
tail --lines=500 /home/yellowtent/platformdata/logs/box.log