diff --git a/package-lock.json b/package-lock.json index 0be20711c..dc093e1b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,7 +43,6 @@ "safetydance": "^2.2.0", "semver": "^7.3.8", "speakeasy": "^2.0.0", - "split": "^1.0.1", "superagent": "^7.1.5", "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error", "tldjs": "^2.3.1", @@ -4259,17 +4258,6 @@ "node": ">= 0.10.0" } }, - "node_modules/split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", - "dependencies": { - "through": "2" - }, - "engines": { - "node": "*" - } - }, "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -4574,11 +4562,6 @@ "node": ">= 6" } }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "node_modules/tldjs": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/tldjs/-/tldjs-2.3.1.tgz", @@ -8500,14 +8483,6 @@ "base32.js": "0.0.1" } }, - "split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", - "requires": { - "through": "2" - } - }, "split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -8750,11 +8725,6 @@ } } }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "tldjs": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/tldjs/-/tldjs-2.3.1.tgz", diff --git a/package.json b/package.json index f0e4a6128..e5c296a28 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "safetydance": "^2.2.0", "semver": "^7.3.8", "speakeasy": "^2.0.0", - "split": "^1.0.1", "superagent": "^7.1.5", "tar-fs": "github:cloudron-io/tar-fs#ignore_stat_error", "tldjs": "^2.3.1", diff --git a/src/apps.js b/src/apps.js index 03491042d..d88451283 100644 --- a/src/apps.js +++ b/src/apps.js @@ -160,6 +160,7 @@ const appstore = require('./appstore.js'), domains = require('./domains.js'), eventlog = require('./eventlog.js'), fs = require('fs'), + LogStream = require('./log-stream.js'), mail = require('./mail.js'), manifestFormat = require('cloudron-manifestformat'), mounts = require('./mounts.js'), @@ -175,7 +176,6 @@ const appstore = require('./appstore.js'), settings = require('./settings.js'), shell = require('./shell.js'), spawn = require('child_process').spawn, - split = require('split'), storage = require('./storage.js'), superagent = require('superagent'), system = require('./system.js'), @@ -2026,29 +2026,12 @@ async function getLogs(app, options) { const logPaths = await getLogPaths(app); const cp = spawn('/usr/bin/tail', args.concat(logPaths)); - const transformStream = split(function mapper(line) { - if (format !== 'json') return line + '\n'; + const logStream = new LogStream({ format, source: appId }); + logStream.close = cp.kill.bind(cp, 'SIGKILL'); // hook for caller. closing stream kills the child process - const data = line.split(' '); // logs are - let timestamp = (new Date(data[0])).getTime(); - if (isNaN(timestamp)) timestamp = 0; - const message = line.slice(data[0].length+1); + cp.stdout.pipe(logStream); - // ignore faulty empty logs - if (!timestamp && !message) return; - - return JSON.stringify({ - realtimeTimestamp: timestamp * 1000, - message: message, - source: appId - }) + '\n'; - }); - - transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process - - cp.stdout.pipe(transformStream); - - return transformStream; + return logStream; } // never fails just prints error diff --git a/src/cloudron.js b/src/cloudron.js index 2945a1a28..af07b1a59 100644 --- a/src/cloudron.js +++ b/src/cloudron.js @@ -39,6 +39,7 @@ const apps = require('./apps.js'), domains = require('./domains.js'), eventlog = require('./eventlog.js'), fs = require('fs'), + LogStream = require('./log-stream.js'), mail = require('./mail.js'), notifications = require('./notifications.js'), path = require('path'), @@ -50,7 +51,6 @@ const apps = require('./apps.js'), settings = require('./settings.js'), shell = require('./shell.js'), spawn = require('child_process').spawn, - split = require('split'), sysinfo = require('./sysinfo.js'), tasks = require('./tasks.js'), users = require('./users.js'); @@ -230,25 +230,12 @@ async function getLogs(unit, options) { const cp = spawn('/usr/bin/tail', args); - const transformStream = split(function mapper(line) { - if (format !== 'json') return line + '\n'; + const logStream = new LogStream({ format, source: unit }); + logStream.close = cp.kill.bind(cp, 'SIGKILL'); // hook for caller. closing stream kills the child process - const data = line.split(' '); // logs are - let timestamp = (new Date(data[0])).getTime(); - if (isNaN(timestamp)) timestamp = 0; + cp.stdout.pipe(logStream); - return JSON.stringify({ - realtimeTimestamp: timestamp * 1000, - message: line.slice(data[0].length+1), - source: unit - }) + '\n'; - }); - - transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process - - cp.stdout.pipe(transformStream); - - return transformStream; + return logStream; } async function prepareDashboardDomain(domain, auditSource) { diff --git a/src/log-stream.js b/src/log-stream.js new file mode 100644 index 000000000..33097179a --- /dev/null +++ b/src/log-stream.js @@ -0,0 +1,52 @@ +'use strict'; + +const stream = require('stream'), + { StringDecoder } = require('string_decoder'), + TransformStream = stream.Transform; + +class LogStream extends TransformStream { + constructor(options) { + super(); + this._options = Object.assign({ source: 'unknown', format: 'json' }, options); + this._decoder = new StringDecoder(); + this._soFar = ''; + } + + _format(line) { + if (this._options.format !== 'json') return line + '\n'; + + const data = line.split(' '); // logs are + let timestamp = (new Date(data[0])).getTime(); + if (isNaN(timestamp)) timestamp = 0; + const message = line.slice(data[0].length+1); + + // ignore faulty empty logs + if (!timestamp && !message) return; + + return JSON.stringify({ + realtimeTimestamp: timestamp * 1000, + message: message, + source: this._options.source + }) + '\n'; + } + + _transform(chunk, encoding, callback) { + const data = this._soFar + this._decoder.write(chunk); + let start = this._soFar.length, end = -1; + while ((end = data.indexOf('\n', start)) !== -1) { + const line = data.slice(start, end); // does not include end + this.push(this._format(line)); + start = end + 1; + } + this._soFar = data.slice(start); + callback(null); + } + + _flush(callback) { + const line = this._soFar + this._decoder.end(); + this.push(this._format(line)); + callback(null); + } +} + +exports = module.exports = LogStream; diff --git a/src/services.js b/src/services.js index 6e4e7f162..a5e271677 100644 --- a/src/services.js +++ b/src/services.js @@ -44,6 +44,7 @@ const addonConfigs = require('./addonconfigs.js'), hat = require('./hat.js'), http = require('http'), infra = require('./infra_version.js'), + LogStream = require('./log-stream.js'), mail = require('./mail.js'), os = require('os'), path = require('path'), @@ -56,7 +57,6 @@ const addonConfigs = require('./addonconfigs.js'), sftp = require('./sftp.js'), shell = require('./shell.js'), spawn = require('child_process').spawn, - split = require('split'), superagent = require('superagent'), system = require('./system.js'); @@ -471,29 +471,12 @@ async function getServiceLogs(id, options) { const cp = spawn(cmd, args); - const transformStream = split(function mapper(line) { - if (format !== 'json') return line + '\n'; + const logStream = new LogStream({ format, source: name }); + logStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process - const data = line.split(' '); // logs are - let timestamp = (new Date(data[0])).getTime(); - if (isNaN(timestamp)) timestamp = 0; - const message = line.slice(data[0].length+1); + cp.stdout.pipe(logStream); - // ignore faulty empty logs - if (!timestamp && !message) return; - - return JSON.stringify({ - realtimeTimestamp: timestamp * 1000, - message: message, - source: name - }) + '\n'; - }); - - transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process - - cp.stdout.pipe(transformStream); - - return transformStream; + return logStream; } async function rebuildService(id, auditSource) { diff --git a/src/tasks.js b/src/tasks.js index 8721e039e..1f86c11ba 100644 --- a/src/tasks.js +++ b/src/tasks.js @@ -46,12 +46,12 @@ const assert = require('assert'), BoxError = require('./boxerror.js'), database = require('./database.js'), debug = require('debug')('box:tasks'), + LogStream = require('./log-stream.js'), path = require('path'), paths = require('./paths.js'), safe = require('safetydance'), shell = require('./shell.js'), spawn = require('child_process').spawn, - split = require('split'), _ = require('underscore'); let gTasks = {}; // indexed by task id @@ -281,29 +281,12 @@ function getLogs(taskId, options) { const cp = spawn(cmd, args); - const transformStream = split(function mapper(line) { - if (format !== 'json') return line + '\n'; + const logStream = new LogStream({ format, source: taskId }); + logStream.close = cp.kill.bind(cp, 'SIGKILL'); // hook for caller. closing stream kills the child process - const data = line.split(' '); // logs are - let timestamp = (new Date(data[0])).getTime(); - if (isNaN(timestamp)) timestamp = 0; - const message = line.slice(data[0].length+1); + cp.stdout.pipe(logStream); - // ignore faulty empty logs - if (!timestamp && !message) return; - - return JSON.stringify({ - realtimeTimestamp: timestamp * 1000, - message: message, - source: taskId - }) + '\n'; - }); - - transformStream.close = cp.kill.bind(cp, 'SIGKILL'); // closing stream kills the child process - - cp.stdout.pipe(transformStream); - - return transformStream; + return logStream; } // removes all fields that are strictly private and should never be returned by API calls