01d0c738bc
mostly we want trace() and log(). trace() can be enabled whenever we want by flipping a flag and restarting box
505 lines
22 KiB
JavaScript
505 lines
22 KiB
JavaScript
import apps from './apps.js';
|
|
import assert from 'node:assert';
|
|
import BoxError from './boxerror.js';
|
|
import constants from './constants.js';
|
|
import logger from './logger.js';
|
|
import docker from './docker.js';
|
|
import fs from 'node:fs';
|
|
import net from 'node:net';
|
|
import network from './network.js';
|
|
import os from 'node:os';
|
|
import { Readable } from 'node:stream';
|
|
import safe from 'safetydance';
|
|
import services from './services.js';
|
|
import shellModule from './shell.js';
|
|
import superagent from '@cloudron/superagent';
|
|
import _ from './underscore.js';
|
|
|
|
const { log, trace } = logger('metrics');
|
|
const shell = shellModule('metrics');
|
|
|
|
|
|
function translateContainerStatsSync(stats) {
|
|
assert.strictEqual(typeof stats, 'object');
|
|
|
|
// the container is missing or stopped. better not to inspect and check State since a race is possible
|
|
if (Object.keys(stats.pids_stats || {}) === 0 || Object.keys(stats.memory_stats || {}).length === 0 || stats.blkio_stats.io_service_bytes_recursive === null) return null;
|
|
|
|
const networkRead = stats.networks ? stats.networks.eth0.rx_bytes : 0; // in host mode (turn), networks is missing
|
|
const networkWrite = stats.networks ? stats.networks.eth0.tx_bytes : 0; // in host mode (turn), networks is missing
|
|
|
|
const memoryUsed = stats.memory_stats.usage;
|
|
const memoryMax = stats.memory_stats.limit;
|
|
|
|
const blkioStats = stats.blkio_stats.io_service_bytes_recursive;
|
|
const blockRead = blkioStats.filter(entry => entry.op === 'read').reduce((sum, entry) => sum + entry.value, 0);
|
|
const blockWrite = blkioStats.filter(entry => entry.op === 'write').reduce((sum, entry) => sum + entry.value, 0);
|
|
|
|
const cpuUsageMsecs = stats.cpu_stats.cpu_usage.total_usage / 1e6; // convert from nano to msecs (to match system metrics)
|
|
const systemUsageMsecs = stats.cpu_stats.system_cpu_usage / 1e6;
|
|
|
|
const pidCount = stats.pids_stats.current;
|
|
|
|
return { ts: new Date(stats.read), pidCount, networkRead, networkWrite, blockRead, blockWrite, memoryUsed, memoryMax, cpuUsageMsecs, systemUsageMsecs };
|
|
}
|
|
|
|
async function readContainerMetrics() {
|
|
const allAddons = [ 'turn', 'mail', 'mongodb', 'mysql', 'postgresql' ];
|
|
|
|
const containerNames = allAddons;
|
|
for (const app of await apps.list()) {
|
|
if (app.containerId) containerNames.push(app.id); // containerId can be null if app is installing. metrics must be stored by appId since container id changes over time
|
|
if (app.manifest.addons?.redis && app.enableRedis) containerNames.push(`redis-${app.id}`);
|
|
}
|
|
|
|
const metrics = {};
|
|
for (const containerName of containerNames) {
|
|
const [error, stats] = await safe(docker.getStats(containerName, { stream: false }));
|
|
if (error) continue;
|
|
|
|
const translated = translateContainerStatsSync(stats);
|
|
if (translated) metrics[containerName] = translated;
|
|
}
|
|
|
|
return metrics;
|
|
}
|
|
|
|
async function readMemoryMetrics() {
|
|
const output = await fs.promises.readFile('/proc/meminfo', { encoding: 'utf8' });
|
|
|
|
const totalMemoryMatch = output.match(/^MemTotal:\s+(\d+)/m);
|
|
const freeMemoryMatch = output.match(/^MemFree:\s+(\d+)/m);
|
|
const buffersMatch = output.match(/^Buffers:\s+(\d+)/m);
|
|
const cachedMatch = output.match(/^Cached:\s+(\d+)/m);
|
|
|
|
if (!totalMemoryMatch || !freeMemoryMatch || !buffersMatch || !cachedMatch) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not find memory used');
|
|
|
|
const memoryUsed = parseInt(totalMemoryMatch[1]) * 1024 - parseInt(freeMemoryMatch[1]) * 1024 - parseInt(buffersMatch[1]) * 1024 - parseInt(cachedMatch[1]) * 1024;
|
|
|
|
const swapTotalMatch = output.match(/^SwapTotal:\s+(\d+)/m);
|
|
const swapFreeMatch = output.match(/^SwapFree:\s+(\d+)/m);
|
|
|
|
if (!swapTotalMatch || !swapFreeMatch) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Could not find swap used');
|
|
|
|
const swapUsed = parseInt(swapTotalMatch[1]) * 1024 - parseInt(swapFreeMatch[1]) * 1024;
|
|
|
|
return {
|
|
memoryUsed,
|
|
swapUsed
|
|
};
|
|
}
|
|
|
|
async function readCpuMetrics() {
|
|
const cpus = os.cpus();
|
|
const userMsecs = cpus.map(c => c.times.user).reduce((p, c) => p+c);
|
|
const sysMsecs = cpus.map(c => c.times.sys).reduce((p, c) => p+c);
|
|
|
|
return { userMsecs, sysMsecs }; // these values are the times spent since system start
|
|
}
|
|
|
|
let gRootDiskName = null;
|
|
async function getRootDiskName() {
|
|
if (gRootDiskName) return gRootDiskName;
|
|
|
|
const mounts = await fs.promises.readFile('/proc/mounts', { encoding: 'utf8' });
|
|
const rootfsLine = mounts.split('\n').find(line => line.split(/\s+/)[1] === '/');
|
|
if (!rootfsLine) throw new BoxError(BoxError.EXTERNAL_ERROR, 'Root mount not found');
|
|
const maybeDevicePath = rootfsLine.split(/\s+/)[0]; // eg. /dev/foo . This can be a partition or a disk path
|
|
|
|
// for LVM, the canonical devicePath might be a symlink to the real disk
|
|
const devicePath = await fs.promises.realpath(maybeDevicePath);
|
|
|
|
// keep going up to find the final parent disk
|
|
let pkname = devicePath.replace('/dev/', '');
|
|
while (true) {
|
|
// -n is no headings , -d is no holder devices or slaves , -o is output format . PKNAME is parent kernel name
|
|
const output = await shell.spawn('lsblk', [ '-ndo', 'PKNAME', `/dev/${pkname}` ], { encoding: 'utf8' });
|
|
if (!output.trim()) break;
|
|
pkname = output.trim();
|
|
}
|
|
|
|
gRootDiskName = pkname;
|
|
return gRootDiskName;
|
|
}
|
|
|
|
async function readDiskMetrics() {
|
|
const [rootDiskError, rootDiskName] = await safe(getRootDiskName());
|
|
if (rootDiskError) throw new BoxError(BoxError.EXTERNAL_ERROR, `Could not detect root disk: ${rootDiskError.message}`);
|
|
const diskstats = await fs.promises.readFile('/proc/diskstats', { encoding: 'utf8' });
|
|
const statsLine = diskstats.split('\n').find(l => l.includes(` ${rootDiskName} `));
|
|
if (!statsLine) throw new BoxError(BoxError.EXTERNAL_ERROR, `Could not get disk stats of ${rootDiskName}`);
|
|
|
|
const parts = statsLine.trim().split(/\s+/);
|
|
const sectorsRead = parseInt(parts[5], 10); // field 6 . one sector is 512 bytes
|
|
const sectorsWrite = parseInt(parts[9], 10); // field 10
|
|
const blockRead = sectorsRead * 512;
|
|
const blockWrite = sectorsWrite * 512;
|
|
|
|
return { blockRead, blockWrite };
|
|
}
|
|
|
|
async function readNetworkMetrics() {
|
|
const defaultIface = await network.getDefaultInterface();
|
|
|
|
const [rx, tx] = await Promise.all([
|
|
fs.promises.readFile(`/sys/class/net/${defaultIface}/statistics/rx_bytes`, { encoding: 'utf8' }),
|
|
fs.promises.readFile(`/sys/class/net/${defaultIface}/statistics/tx_bytes`, { encoding: 'utf8' })
|
|
]);
|
|
|
|
return {
|
|
networkRead: parseInt(rx.trim(), 10),
|
|
networkWrite: parseInt(tx.trim(), 10)
|
|
};
|
|
}
|
|
|
|
async function readSystemMetrics() {
|
|
const memoryMetrics = await readMemoryMetrics();
|
|
const cpuMetrics = await readCpuMetrics();
|
|
const diskMetrics = await readDiskMetrics();
|
|
const networkMetrics = await readNetworkMetrics();
|
|
|
|
// { memoryUsed, swapUsed, userMsecs, sysMsecs, blockRead, blockWrite, networkRead, networkWrite }
|
|
return { ts: new Date(), ...memoryMetrics, ...cpuMetrics, ...diskMetrics, ...networkMetrics };
|
|
}
|
|
|
|
async function sendToGraphite() {
|
|
// log('sendStatsToGraphite: collecting stats');
|
|
|
|
const result = await readSystemMetrics();
|
|
|
|
const graphiteMetrics = [
|
|
{ path: `cloudron.system.memory-used`, value: result.memoryUsed },
|
|
{ path: `cloudron.system.swap-used`, value: result.swapUsed },
|
|
{ path: `cloudron.system.cpu-user`, value: result.userMsecs },
|
|
{ path: `cloudron.system.cpu-sys`, value: result.sysMsecs },
|
|
{ path: `cloudron.system.blockio-read`, value: result.blockRead },
|
|
{ path: `cloudron.system.blockio-write`, value: result.blockWrite },
|
|
{ path: `cloudron.system.network-read`, value: result.networkRead },
|
|
{ path: `cloudron.system.network-write`, value: result.networkWrite }
|
|
];
|
|
|
|
const dockerMetrics = await readContainerMetrics();
|
|
for (const [name, value] of Object.entries(dockerMetrics)) {
|
|
graphiteMetrics.push(
|
|
{ path: `cloudron.container-${name}.network-read`, value: value.networkRead },
|
|
{ path: `cloudron.container-${name}.network-write`, value: value.networkWrite },
|
|
{ path: `cloudron.container-${name}.blockio-read`, value: value.blockRead },
|
|
{ path: `cloudron.container-${name}.blockio-write`, value: value.blockWrite },
|
|
{ path: `cloudron.container-${name}.memory-used`, value: value.memoryUsed },
|
|
{ path: `cloudron.container-${name}.memory-max`, value: value.memoryMax },
|
|
{ path: `cloudron.container-${name}.cpu-usage`, value: value.cpuUsageMsecs },
|
|
);
|
|
}
|
|
|
|
return new Promise((resolve) => {
|
|
const client = new net.Socket();
|
|
client.connect(constants.GRAPHITE_PORT, constants.GRAPHITE_SERVICE_IPv4, () => {
|
|
const now = Math.floor(Date.now() / 1000);
|
|
|
|
for (const metric of graphiteMetrics) {
|
|
client.write(`${metric.path} ${metric.value} ${now}\n`);
|
|
}
|
|
client.end();
|
|
});
|
|
|
|
client.on('error', (error) => {
|
|
log(`Error sending data to graphite: ${error.message}`);
|
|
resolve();
|
|
});
|
|
|
|
client.on('end', () => resolve());
|
|
});
|
|
}
|
|
|
|
// for testing locally: curl 'http://${graphite-ip}:8000/graphite-web/render?format=json&from=-1min&target=absolute(collectd.localhost.du-docker.capacity-usage)'
|
|
// the datapoint is (value, timestamp) https://graphite.readthedocs.io/en/latest/
|
|
async function getGraphiteUrl() {
|
|
const [error, result] = await safe(docker.inspect('graphite'));
|
|
if (error && error.reason === BoxError.NOT_FOUND) return { status: services.SERVICE_STATUS_ERROR };
|
|
if (error) throw error;
|
|
|
|
const ip = safe.query(result, 'NetworkSettings.Networks.cloudron.IPAddress', null);
|
|
if (!ip) throw new BoxError(BoxError.INACTIVE, 'Error getting IP of graphite service');
|
|
|
|
return `http://${ip}:8000/graphite-web/render`;
|
|
}
|
|
|
|
async function getContainer(name, options) {
|
|
assert.strictEqual(typeof name, 'string');
|
|
assert.strictEqual(typeof options, 'object');
|
|
|
|
const { fromSecs, intervalSecs, noNullPoints } = options;
|
|
|
|
const graphiteUrl = await getGraphiteUrl();
|
|
|
|
const targets = [
|
|
// perSecond is nonNegativeDerivative over time . this value is the cpu usage in msecs .
|
|
// (cpu usage msecs) / (cpus * 1000) is the percent but over all cpus. times 100 is the percent.
|
|
// but the y-scale is cpus times 100. so, we only need to scale by 0.1
|
|
`scale(perSecond(cloudron.container-${name}.cpu-usage),0.1)`,
|
|
`summarize(cloudron.container-${name}.memory-used, "${intervalSecs}s", "avg")`,
|
|
|
|
// get the rate in interval window
|
|
`summarize(perSecond(cloudron.container-${name}.blockio-read), "${intervalSecs}s", "avg")`,
|
|
`summarize(perSecond(cloudron.container-${name}.blockio-write), "${intervalSecs}s", "avg")`,
|
|
`summarize(perSecond(cloudron.container-${name}.network-read), "${intervalSecs}s", "avg")`,
|
|
`summarize(perSecond(cloudron.container-${name}.network-write), "${intervalSecs}s", "avg")`,
|
|
|
|
// just get the max in interval window for absolute numbers
|
|
`summarize(cloudron.container-${name}.blockio-read, "${intervalSecs}s", "max")`,
|
|
`summarize(cloudron.container-${name}.blockio-write, "${intervalSecs}s", "max")`,
|
|
`summarize(cloudron.container-${name}.network-read, "${intervalSecs}s", "max")`,
|
|
`summarize(cloudron.container-${name}.network-write, "${intervalSecs}s", "max")`,
|
|
];
|
|
|
|
const results = [];
|
|
|
|
for (const target of targets) {
|
|
const query = {
|
|
target,
|
|
format: 'json',
|
|
from: `-${fromSecs}s`,
|
|
until: 'now+20s', // until is exclusive. 'now' is otherwise not included
|
|
noNullPoints: !!noNullPoints
|
|
};
|
|
|
|
const [error, response] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
|
|
if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
|
|
if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error with ${target}: ${response.status} ${response.text}`);
|
|
|
|
results.push(response.body[0] && response.body[0].datapoints ? response.body[0].datapoints : []);
|
|
}
|
|
|
|
// results are datapoints[[value, ts], [value, ts], ...];
|
|
return {
|
|
cpu: results[0],
|
|
memory: results[1],
|
|
blockReadRate: results[2],
|
|
blockWriteRate: results[3],
|
|
networkReadRate: results[4],
|
|
networkWriteRate: results[5],
|
|
blockReadTotal: results[6].at(-1)?.[0] ?? 0,
|
|
blockWriteTotal: results[7].at(-1)?.[0] ?? 0,
|
|
networkReadTotal: results[8].at(-1)?.[0] ?? 0,
|
|
networkWriteTotal: results[9].at(-1)?.[0] ?? 0,
|
|
};
|
|
}
|
|
|
|
async function readSystemFromGraphite(options) {
|
|
assert.strictEqual(typeof options, 'object');
|
|
|
|
const { fromSecs, intervalSecs, noNullPoints } = options;
|
|
|
|
const graphiteUrl = await getGraphiteUrl();
|
|
|
|
// example: curl 'http://172.18.30.5:8000/graphite-web/render?target=cloudron.system.cpu-user&target=cloudron.system.cpu-sys&format=json&from=-1min&until=now&noNullPoints=false' | python3 -m json.tool
|
|
const targets = [
|
|
// perSecond is nonNegativeDerivative over time . this value is the cpu usage in msecs .
|
|
// (cpu usage msecs) / (cpus * 1000) is the percent but over all cpus. times 100 is the percent.
|
|
// but the y-scale is cpus times 100. so, we only need to scale by 0.1
|
|
`scale(perSecond(sumSeries(cloudron.system.cpu-user,cloudron.system.cpu-sys)),0.1)`,
|
|
`summarize(cloudron.system.memory-used, "${intervalSecs}s", "avg")`,
|
|
`summarize(cloudron.system.swap-used, "${intervalSecs}s", "avg")`,
|
|
|
|
// get the rate in interval window
|
|
`summarize(perSecond(cloudron.system.blockio-read), "${intervalSecs}s", "avg")`,
|
|
`summarize(perSecond(cloudron.system.blockio-write), "${intervalSecs}s", "avg")`,
|
|
`summarize(perSecond(cloudron.system.network-read), "${intervalSecs}s", "avg")`,
|
|
`summarize(perSecond(cloudron.system.network-write), "${intervalSecs}s", "avg")`,
|
|
|
|
// just get the max in interval window for absolute numbers
|
|
`summarize(cloudron.system.blockio-read, "${intervalSecs}s", "max")`,
|
|
`summarize(cloudron.system.blockio-write, "${intervalSecs}s", "max")`,
|
|
`summarize(cloudron.system.network-read, "${intervalSecs}s", "max")`,
|
|
`summarize(cloudron.system.network-write, "${intervalSecs}s", "max")`,
|
|
|
|
];
|
|
|
|
const results = [];
|
|
|
|
for (const target of targets) {
|
|
const query = {
|
|
target,
|
|
format: 'json',
|
|
from: `-${fromSecs}s`,
|
|
until: 'now',
|
|
noNullPoints: !!noNullPoints
|
|
};
|
|
|
|
const [error, response] = await safe(superagent.get(graphiteUrl).query(query).timeout(30 * 1000).ok(() => true));
|
|
if (error) throw new BoxError(BoxError.NETWORK_ERROR, error);
|
|
if (response.status !== 200) throw new BoxError(BoxError.EXTERNAL_ERROR, `Unknown error with ${target}: ${response.status} ${response.text}`);
|
|
|
|
results.push(response.body[0] && response.body[0].datapoints ? response.body[0].datapoints : []);
|
|
}
|
|
|
|
return {
|
|
cpu: results[0],
|
|
memory: results[1],
|
|
swap: results[2],
|
|
blockReadRate: results[3],
|
|
blockWriteRate: results[4],
|
|
networkReadRate: results[5],
|
|
networkWriteRate: results[6],
|
|
blockReadTotal: results[7].at(-1)?.[0] ?? 0,
|
|
blockWriteTotal: results[8].at(-1)?.[0] ?? 0,
|
|
networkReadTotal: results[9].at(-1)?.[0] ?? 0,
|
|
networkWriteTotal: results[10].at(-1)?.[0] ?? 0,
|
|
};
|
|
}
|
|
|
|
// CPU: stress --cpu 2 --timeout 60
|
|
// Memory: stress --vm 2 --vm-bytes 256M
|
|
// Network:
|
|
// raw stats: ip -s link show eth0
|
|
// testing: curl -o /dev/null https://ash-speed.hetzner.com/10GB.bin and then use nethogs eth0 (cycle with 'm')
|
|
// Disk:
|
|
// writing: fio --name=rate-test --filename=tempfile --rw=write --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
|
// reading: fio --name=rate-test --filename=tempfile --rw=read --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
|
async function get(options) {
|
|
assert.strictEqual(typeof options, 'object');
|
|
|
|
const result = {};
|
|
|
|
if (options.system) result.system = await readSystemFromGraphite(options);
|
|
|
|
for (const appId of (options.appIds || [])) {
|
|
result[appId] = await getContainer(appId, options);
|
|
}
|
|
|
|
for (const serviceId of (options.serviceIds || [])) {
|
|
result[serviceId] = await getContainer(serviceId, options);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
async function pipeContainerToMap(name, statsMap) {
|
|
assert.strictEqual(typeof name, 'string');
|
|
assert.ok(statsMap instanceof Map);
|
|
|
|
// we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups
|
|
const statsStream = await docker.getStats(name, { stream: true });
|
|
|
|
statsStream.on('error', (error) => log(error));
|
|
statsStream.on('data', (data) => {
|
|
const stats = JSON.parse(data.toString('utf8'));
|
|
const metrics = translateContainerStatsSync(stats);
|
|
if (!metrics) { // maybe the container stopped
|
|
statsMap.delete(name);
|
|
return;
|
|
}
|
|
|
|
const { ts, networkRead, networkWrite, blockRead, blockWrite, memoryUsed, cpuUsageMsecs } = metrics;
|
|
|
|
const oldMetrics = statsMap.get(name)?.raw || null;
|
|
|
|
const gap = oldMetrics ? (ts - oldMetrics.ts) : null;
|
|
|
|
const cpuPercent = oldMetrics ? (cpuUsageMsecs - oldMetrics.cpuUsageMsecs) * 100 / gap : null;
|
|
const blockReadRate = oldMetrics ? (blockRead - oldMetrics.blockRead) / (gap/1000) : null;
|
|
const blockWriteRate = oldMetrics ? (blockWrite - oldMetrics.blockWrite) / (gap/1000) : null;
|
|
const networkReadRate = oldMetrics ? (networkRead - oldMetrics.networkRead) / (gap/1000) : null;
|
|
const networkWriteRate = oldMetrics ? (networkWrite - oldMetrics.networkWrite) / (gap/1000) : null;
|
|
|
|
const nowSecs = ts.getTime() / 1000; // convert to secs to match graphite return value but this is thrown away and patched during streaming time
|
|
statsMap.set(name, {
|
|
raw: metrics,
|
|
cpu: [ cpuPercent, nowSecs ],
|
|
memory: [ memoryUsed, nowSecs ],
|
|
|
|
blockReadRate: [ blockReadRate, nowSecs ],
|
|
blockWriteRate: [ blockWriteRate, nowSecs ],
|
|
blockReadTotal: metrics.blockRead,
|
|
blockWriteTotal: metrics.blockWrite,
|
|
|
|
networkReadRate: [ networkReadRate, nowSecs ],
|
|
networkWriteRate: [ networkWriteRate, nowSecs ],
|
|
networkReadTotal: metrics.networkRead,
|
|
networkWriteTotal: metrics.networkWrite,
|
|
});
|
|
});
|
|
|
|
return statsStream;
|
|
}
|
|
|
|
async function pipeSystemToMap(statsMap) {
|
|
assert.ok(statsMap instanceof Map);
|
|
|
|
const metrics = await readSystemMetrics();
|
|
const oldMetrics = statsMap.get('system')?.raw || null;
|
|
|
|
const gap = oldMetrics ? (metrics.ts - oldMetrics.ts) : null;
|
|
|
|
const cpuPercent = oldMetrics ? (metrics.userMsecs + metrics.sysMsecs - oldMetrics.userMsecs - oldMetrics.sysMsecs) * 100 / gap : null;
|
|
const blockReadRate = oldMetrics ? (metrics.blockRead - oldMetrics.blockRead) / (gap/1000) : null;
|
|
const blockWriteRate = oldMetrics ? (metrics.blockWrite - oldMetrics.blockWrite) / (gap/1000) : null;
|
|
const networkReadRate = oldMetrics ? (metrics.networkRead - oldMetrics.networkRead) / (gap/1000) : null;
|
|
const networkWriteRate = oldMetrics ? (metrics.networkWrite - oldMetrics.networkWrite) / (gap/1000) : null;
|
|
|
|
const nowSecs = Date.now() / 1000; // convert to secs to match graphite return value but this is thrown away and patched during streaming time
|
|
const systemStats = {
|
|
raw: metrics,
|
|
cpu: [ cpuPercent, nowSecs ],
|
|
memory: [ metrics.memoryUsed, nowSecs ],
|
|
swap: [ metrics.swapUsed, nowSecs ],
|
|
|
|
blockReadRate: [ blockReadRate, nowSecs ],
|
|
blockWriteRate: [ blockWriteRate, nowSecs ],
|
|
blockReadTotal: metrics.blockRead,
|
|
blockWriteTotal: metrics.blockWrite,
|
|
|
|
networkReadRate: [ networkReadRate, nowSecs ],
|
|
networkWriteRate: [ networkWriteRate, nowSecs ],
|
|
networkReadTotal: metrics.networkRead,
|
|
networkWriteTotal: metrics.networkWrite,
|
|
};
|
|
statsMap.set('system', systemStats);
|
|
}
|
|
|
|
async function getStream(options) {
|
|
assert.strictEqual(typeof options, 'object');
|
|
|
|
const statsMap = new Map();
|
|
let intervalId = null, containerStreamPromises = [];
|
|
|
|
const metricsStream = new Readable({
|
|
objectMode: true,
|
|
read(/*size*/) { /* ignored, we push via interval */ },
|
|
destroy(error, callback) {
|
|
clearInterval(intervalId);
|
|
containerStreamPromises.forEach(cs => { if (cs.status === 'fulfilled' && cs.value) cs.value.destroy(error); });
|
|
callback(error);
|
|
}
|
|
});
|
|
|
|
const containerNames = (options.appIds || []).concat(options.serviceIds || []);
|
|
containerStreamPromises = await Promise.allSettled(containerNames.map(containerName => pipeContainerToMap(containerName, statsMap)));
|
|
|
|
const INTERVAL_MSECS = 1000;
|
|
intervalId = setInterval(async () => {
|
|
if (options.system) await safe(pipeSystemToMap(statsMap), { debug: log });
|
|
|
|
const result = {};
|
|
const nowSecs = Date.now() / 1000; // to match graphite return value
|
|
|
|
// patch the stats to have the current timestamp
|
|
for (const [id, stats] of statsMap) {
|
|
result[id] = structuredClone(_.omit(stats, ['raw']));
|
|
for (const statValue of Object.values(result[id])) {
|
|
if (Array.isArray(statValue)) statValue[1] = nowSecs;
|
|
}
|
|
}
|
|
|
|
metricsStream.push(result);
|
|
}, INTERVAL_MSECS);
|
|
|
|
return metricsStream;
|
|
}
|
|
|
|
export default {
|
|
get,
|
|
getStream,
|
|
sendToGraphite
|
|
};
|