metrics: overlay app metrics over system metrics
This commit is contained in:
@@ -21,21 +21,13 @@ const props = defineProps({
|
||||
title: String,
|
||||
subtext: String,
|
||||
period: Object, // { hours, format, tooltpFormat }
|
||||
datasetLabels: {
|
||||
type: Array,
|
||||
validator: (val) => Array.isArray(val) && val.every(item => typeof item === 'string')
|
||||
},
|
||||
datasetColors: {
|
||||
type: Array,
|
||||
validator: (val) => Array.isArray(val) && val.every(item => typeof item === 'string')
|
||||
},
|
||||
yscale: String, // cpu, memory
|
||||
memory: Number,
|
||||
cpuCount: Number,
|
||||
highMark: Number,
|
||||
});
|
||||
|
||||
function createGraphOptions({ yscale, period, displayLegend, highMark }) {
|
||||
function createGraphOptions({ yscale, period, highMark }) {
|
||||
let startTime, endTime, stepSize, count; // x axis configuration values
|
||||
|
||||
const now = Date.now();
|
||||
@@ -54,8 +46,8 @@ function createGraphOptions({ yscale, period, displayLegend, highMark }) {
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
display: displayLegend,
|
||||
position: 'bottom'
|
||||
display: false,
|
||||
position: 'bottom' // not used, hidden since color code is shown in tooltip
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
@@ -121,10 +113,23 @@ function transformData(data) {
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
function setData(...data) {
|
||||
for (const [index, items] of data.entries()) {
|
||||
graph.data.datasets[index].data = items.map(transformData);
|
||||
function setDatasets(datasets) {
|
||||
graph.data = { datasets: [] };
|
||||
|
||||
for (const dataset of datasets) {
|
||||
graph.data.datasets.push({
|
||||
label: dataset.label,
|
||||
data: dataset.data.map(transformData),
|
||||
pointRadius: 0,
|
||||
borderWidth: 1, // https://www.chartjs.org/docs/latest/charts/line.html#line-styling
|
||||
tension: 0.4,
|
||||
showLine: true,
|
||||
fill: true,
|
||||
color: dataset.color,
|
||||
stack: dataset.stack || 'stackgroup', // put them all in same stackgroup
|
||||
});
|
||||
}
|
||||
|
||||
graph.update('none');
|
||||
}
|
||||
|
||||
@@ -140,10 +145,10 @@ function advance() {
|
||||
graph.update('none');
|
||||
}
|
||||
|
||||
function pushData(...data) {
|
||||
function pushData(datasetIndex, ...data) {
|
||||
for (const [index, item] of data.entries()) {
|
||||
graph.data.datasets[index].data.push(transformData(item));
|
||||
pruneGraphData(graph.data.datasets[index], graph.options);
|
||||
graph.data.datasets[datasetIndex+index].data.push(transformData(item));
|
||||
pruneGraphData(graph.data.datasets[datasetIndex+index], graph.options);
|
||||
}
|
||||
graph.update('none');
|
||||
}
|
||||
@@ -154,22 +159,6 @@ function onPeriodChanged() {
|
||||
liveRefreshIntervalId = null;
|
||||
}
|
||||
|
||||
const data = { datasets: [] };
|
||||
|
||||
for (const [index, label] of props.datasetLabels.entries()) {
|
||||
data.datasets.push({
|
||||
label: label,
|
||||
data: [],
|
||||
pointRadius: 0,
|
||||
borderWidth: 1, // https://www.chartjs.org/docs/latest/charts/line.html#line-styling
|
||||
tension: 0.4,
|
||||
showLine: true,
|
||||
fill: true,
|
||||
color: props.datasetColors[index],
|
||||
stack: 'stackgroup' // put them all in same stackgroup
|
||||
});
|
||||
}
|
||||
|
||||
// CPU and Memory graph have known min/max set and auto-scaling gets disabled
|
||||
// Disk and Network graphs auto-scale the y values.
|
||||
|
||||
@@ -236,12 +225,11 @@ function onPeriodChanged() {
|
||||
}
|
||||
|
||||
// this sets a min 'x' based on current timestamp. so it has to re-created every time the period changes
|
||||
const graphOptions = createGraphOptions({ yscale, period: props.period, displayLegend: props.datasetLabels.length > 1, highMark });
|
||||
const graphOptions = createGraphOptions({ yscale, period: props.period, highMark });
|
||||
|
||||
if (!graph) {
|
||||
graph = new Chart(graphNode.value, { type: 'line', data, options: graphOptions });
|
||||
graph = new Chart(graphNode.value, { type: 'line', data: { datasets: [] }, options: graphOptions });
|
||||
} else {
|
||||
graph.data = data;
|
||||
graph.options = graphOptions;
|
||||
graph.update('none');
|
||||
}
|
||||
@@ -261,7 +249,7 @@ onUnmounted(async function () {
|
||||
});
|
||||
|
||||
defineExpose({
|
||||
setData,
|
||||
setDatasets,
|
||||
pushData,
|
||||
});
|
||||
|
||||
|
||||
@@ -4,14 +4,16 @@ import { useI18n } from 'vue-i18n';
|
||||
const i18n = useI18n();
|
||||
const t = i18n.t;
|
||||
|
||||
import { ref, onMounted, onUnmounted, useTemplateRef, nextTick } from 'vue';
|
||||
import { SingleSelect } from 'pankow';
|
||||
import { ref, onMounted, onUnmounted, useTemplateRef, nextTick, watch } from 'vue';
|
||||
import { SingleSelect, MultiSelect } from 'pankow';
|
||||
import Section from './Section.vue';
|
||||
import SystemModel from '../models/SystemModel.js';
|
||||
import { prettyDecimalSize } from 'pankow/utils';
|
||||
import GraphItem from './GraphItem.vue';
|
||||
import AppsModel from '../models/AppsModel.js';
|
||||
|
||||
const systemModel = SystemModel.create();
|
||||
const appsModel = AppsModel.create();
|
||||
|
||||
const periods = [
|
||||
{ hours: 0, label: t('app.graphs.period.live'), tickFormat: 'hh:mm A', tooltipFormat: 'hh:mm:ss A' },
|
||||
@@ -36,73 +38,148 @@ const networkWriteTotal = ref(0);
|
||||
const blockReadTotal = ref(0);
|
||||
const blockWriteTotal = ref(0);
|
||||
|
||||
const containers = ref([]);
|
||||
const allContainers = ref([]);
|
||||
|
||||
let systemMemory = {};
|
||||
let systemCpus = {};
|
||||
let metricStream = null;
|
||||
|
||||
async function liveRefresh() {
|
||||
metricStream = await systemModel.getMetricStream();
|
||||
const options = {
|
||||
system: true,
|
||||
appIds: containers.value.map(c => c.id),
|
||||
serviceIds: []
|
||||
};
|
||||
metricStream = await systemModel.getMetricStream(options);
|
||||
metricStream.onerror = (error) => console.log('event stream error:', error);
|
||||
metricStream.onmessage = (message) => {
|
||||
const data = JSON.parse(message.data);
|
||||
|
||||
cpuGraphItem.value.pushData(data.cpu);
|
||||
memoryGraphItem.value.pushData(data.memory, data.swap);
|
||||
diskGraphItem.value.pushData(data.blockReadRate, data.blockWriteRate);
|
||||
networkGraphItem.value.pushData(data.networkReadRate, data.networkWriteRate);
|
||||
for (const [id, metric] of Object.entries(data)) {
|
||||
const idx = id !== 'system' ? containers.value.findIndex(c => c.id === id) : containers.value.length;
|
||||
|
||||
blockReadTotal.value = prettyDecimalSize(data.blockReadTotal);
|
||||
blockWriteTotal.value = prettyDecimalSize(data.blockWriteTotal);
|
||||
networkReadTotal.value = prettyDecimalSize(data.networkReadTotal);
|
||||
networkWriteTotal.value = prettyDecimalSize(data.networkWriteTotal);
|
||||
cpuGraphItem.value.pushData(idx, metric.cpu);
|
||||
memoryGraphItem.value.pushData(idx*2, metric.memory, metric.swap || []); // apps have no swap
|
||||
diskGraphItem.value.pushData(idx*2, metric.blockReadRate, metric.blockWriteRate);
|
||||
networkGraphItem.value.pushData(idx*2, metric.networkReadRate, metric.networkWriteRate);
|
||||
|
||||
if (id === 'system') {
|
||||
blockReadTotal.value = prettyDecimalSize(metric.blockReadTotal);
|
||||
blockWriteTotal.value = prettyDecimalSize(metric.blockWriteTotal);
|
||||
networkReadTotal.value = prettyDecimalSize(metric.networkReadTotal);
|
||||
networkWriteTotal.value = prettyDecimalSize(metric.networkWriteTotal);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function onPeriodChange() {
|
||||
function generateConsistentColors(n, saturation = 90, lightness = 90) {
|
||||
const baseHue = 204; // from #9ad0f5 → hsl(204,82%,78%)
|
||||
const colors = [];
|
||||
const step = 360 / n;
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const hue = Math.round((baseHue + step * i) % 360); // rotate hue, wrap at 360
|
||||
colors.push(`hsl(${hue}, ${saturation}%, ${lightness}%)`);
|
||||
}
|
||||
|
||||
return colors;
|
||||
}
|
||||
|
||||
function createDatasets() {
|
||||
const colors = generateConsistentColors((containers.value.length+1)*2); // 1 for the 'system'
|
||||
|
||||
const datasets = {
|
||||
cpu: [],
|
||||
memory: [],
|
||||
disk: [],
|
||||
network: [],
|
||||
};
|
||||
const appIds = containers.value.map(c => c.id);
|
||||
for (const [idx, id] of appIds.concat(['system']).entries()) { // live stream code depends on this concat order!
|
||||
const prefix = id === 'system' ? 'System' : containers.value[idx].label;
|
||||
|
||||
datasets.cpu.push({ label: `${prefix} CPU`, color: colors[idx*2], stack: `${prefix}-cpu`, data: [] });
|
||||
datasets.memory.push({ label: `${prefix} Memory`, color: colors[idx*2], stack: `${prefix}-memswap`, data: [] });
|
||||
datasets.memory.push({ label: `${prefix} Swap`, color: colors[idx*2 + 1], stack: `${prefix}-memswap`, data: [] });
|
||||
|
||||
datasets.disk.push({ label: `${prefix} Read`, color: colors[idx*2], stack: `${prefix}-read`, data: [] });
|
||||
datasets.disk.push({ label: `${prefix} Write`, color: colors[idx*2 + 1], stack: `${prefix}-write`, prefix, data: [] });
|
||||
|
||||
datasets.network.push({ label: `${prefix} RX`, color: colors[idx*2], stack: `${prefix}-rx`, data: [] });
|
||||
datasets.network.push({ label: `${prefix} TX`, color: colors[idx*2 + 1], stack: `${prefix}-tx`, data: [] });
|
||||
}
|
||||
|
||||
return datasets;
|
||||
}
|
||||
|
||||
async function rebuild() {
|
||||
if (metricStream) {
|
||||
metricStream.close();
|
||||
metricStream = null;
|
||||
}
|
||||
|
||||
const datasets = createDatasets();
|
||||
|
||||
if (period.value.hours !== 0) {
|
||||
const options = {
|
||||
fromSecs: period.value.hours * 60 * 60,
|
||||
intervalSecs: period.value.intervalSecs,
|
||||
system: true,
|
||||
appIds: containers.value.map(c => c.id),
|
||||
serviceIds: []
|
||||
};
|
||||
const [error, metrics] = await systemModel.getMetrics(options);
|
||||
if (error) return console.error(error);
|
||||
|
||||
const appIds = containers.value.map(c => c.id);
|
||||
for (const [idx, id] of appIds.concat(['system']).entries()) {
|
||||
if (!metrics[id]) continue;
|
||||
datasets.cpu[idx].data = metrics[id].cpu;
|
||||
datasets.memory[idx*2].data = metrics[id].memory;
|
||||
datasets.memory[idx*2 + 1].data = metrics[id].swap || []; // apps have no swap
|
||||
datasets.disk[idx*2].data = metrics[id].blockReadRate;
|
||||
datasets.disk[idx*2 + 1].data = metrics[id].blockWriteRate;
|
||||
datasets.network[idx*2].data = metrics[id].networkReadRate;
|
||||
datasets.network[idx*2 + 1].data = metrics[id].networkWriteRate;
|
||||
}
|
||||
|
||||
networkReadTotal.value = prettyDecimalSize(metrics.system.networkReadTotal);
|
||||
networkWriteTotal.value = prettyDecimalSize(metrics.system.networkWriteTotal);
|
||||
blockReadTotal.value = prettyDecimalSize(metrics.system.blockReadTotal);
|
||||
blockWriteTotal.value = prettyDecimalSize(metrics.system.blockWriteTotal);
|
||||
}
|
||||
|
||||
cpuGraphItem.value.setDatasets(datasets.cpu);
|
||||
memoryGraphItem.value.setDatasets(datasets.memory);
|
||||
diskGraphItem.value.setDatasets(datasets.disk);
|
||||
networkGraphItem.value.setDatasets(datasets.network);
|
||||
|
||||
if (period.value.hours === 0) return await liveRefresh();
|
||||
|
||||
const options = {
|
||||
fromSecs: period.value.hours * 60 * 60,
|
||||
intervalSecs: period.value.intervalSecs,
|
||||
system: true,
|
||||
appIds: [],
|
||||
serviceIds: []
|
||||
};
|
||||
const [error, metrics] = await systemModel.getMetrics(options);
|
||||
if (error) return console.error(error);
|
||||
|
||||
cpuGraphItem.value.setData(metrics.system.cpu);
|
||||
memoryGraphItem.value.setData(metrics.system.memory, metrics.system.swap);
|
||||
diskGraphItem.value.setData(metrics.system.blockReadRate, metrics.system.blockWriteRate);
|
||||
networkGraphItem.value.setData(metrics.system.networkReadRate, metrics.system.networkWriteRate);
|
||||
|
||||
networkReadTotal.value = prettyDecimalSize(metrics.system.networkReadTotal);
|
||||
networkWriteTotal.value = prettyDecimalSize(metrics.system.networkWriteTotal);
|
||||
blockReadTotal.value = prettyDecimalSize(metrics.system.blockReadTotal);
|
||||
blockWriteTotal.value = prettyDecimalSize(metrics.system.blockWriteTotal);
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
let error, result;
|
||||
|
||||
[error, result] = await systemModel.memory();
|
||||
if (error) return console.error(error);
|
||||
|
||||
systemMemory = result;
|
||||
|
||||
[error, result] = await systemModel.cpus();
|
||||
if (error) return console.error(error);
|
||||
|
||||
systemCpus = result;
|
||||
|
||||
[error, result] = await appsModel.list();
|
||||
if (error) return console.error(error);
|
||||
result.forEach(a => a.label = (a.label || a.fqdn));
|
||||
allContainers.value = result;
|
||||
containers.value = [];
|
||||
|
||||
busy.value = false;
|
||||
await nextTick();
|
||||
|
||||
await onPeriodChange();
|
||||
await rebuild();
|
||||
});
|
||||
|
||||
onUnmounted(async () => {
|
||||
@@ -114,7 +191,8 @@ onUnmounted(async () => {
|
||||
<template>
|
||||
<Section :title="$t('system.graphs.title')">
|
||||
<template #header-buttons>
|
||||
<SingleSelect @select="onPeriodChange()" v-model="period" :options="periods" option-label="label"/>
|
||||
<MultiSelect @select="rebuild()" v-model="containers" :options="allContainers" option-label="label" :search-threshold="20"/>
|
||||
<SingleSelect @select="rebuild()" v-model="period" :options="periods" option-label="label"/>
|
||||
</template>
|
||||
|
||||
<div class="graphs" v-if="!busy">
|
||||
@@ -123,8 +201,6 @@ onUnmounted(async () => {
|
||||
:subtext='systemCpus.length ? `${systemCpus.length} Core "${systemCpus[0].model}"` : ""'
|
||||
:period="period"
|
||||
yscale="cpu"
|
||||
:dataset-labels="['CPU']"
|
||||
:dataset-colors="['#9ad0f5']"
|
||||
:cpu-count="systemCpus.length"
|
||||
>
|
||||
</GraphItem>
|
||||
@@ -134,8 +210,6 @@ onUnmounted(async () => {
|
||||
:subtext="`RAM: ${prettyDecimalSize(systemMemory.memory)} Swap: ${prettyDecimalSize(systemMemory.swap)}`"
|
||||
:period="period"
|
||||
yscale="memory"
|
||||
:dataset-labels="['Memory', 'Swap']"
|
||||
:dataset-colors="['#9ad0f5', '#ffb1c1']"
|
||||
:memory="systemMemory.memory + systemMemory.swap"
|
||||
>
|
||||
</GraphItem>
|
||||
@@ -145,8 +219,6 @@ onUnmounted(async () => {
|
||||
:subtext="$t('app.graphs.diskIOTotal', { read: blockReadTotal, write: blockWriteTotal })"
|
||||
:period="period"
|
||||
yscale="disk"
|
||||
:dataset-labels="['Read', 'Write']"
|
||||
:dataset-colors="['#9ad0f5', '#ffb1c1']"
|
||||
>
|
||||
</GraphItem>
|
||||
|
||||
@@ -155,8 +227,6 @@ onUnmounted(async () => {
|
||||
:subtext="$t('app.graphs.networkIOTotal', { inbound: networkReadTotal, outbound: networkWriteTotal })"
|
||||
:period="period"
|
||||
yscale="network"
|
||||
:dataset-labels="['RX', 'TX']"
|
||||
:dataset-colors="['#9ad0f5', '#ffb1c1']"
|
||||
>
|
||||
</GraphItem>
|
||||
</div>
|
||||
|
||||
@@ -102,8 +102,16 @@ function create() {
|
||||
if (error || result.status !== 200) return [error || result];
|
||||
return [null, result.body];
|
||||
},
|
||||
async getMetricStream() {
|
||||
return new EventSource(`${API_ORIGIN}/api/v1/system/metricstream?access_token=${accessToken}`);
|
||||
async getMetricStream(options) {
|
||||
const query = [
|
||||
['system', String(!!options.system)],
|
||||
...options.appIds.map(id => ['appId', id]), // multiple appId=xx
|
||||
...options.serviceIds.map(id => ['serviceId', id]), // multiple serviceId=xx
|
||||
['access_token', accessToken]
|
||||
];
|
||||
|
||||
const queryString = new URLSearchParams(query).toString();
|
||||
return new EventSource(`${API_ORIGIN}/api/v1/system/metricstream?${queryString}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
+83
-71
@@ -1,12 +1,8 @@
|
||||
'use strict';
|
||||
|
||||
exports = module.exports = {
|
||||
getSystem,
|
||||
getSystemStream,
|
||||
|
||||
getContainer,
|
||||
getContainerStream,
|
||||
|
||||
get,
|
||||
getStream,
|
||||
sendToGraphite
|
||||
};
|
||||
|
||||
@@ -22,7 +18,6 @@ const apps = require('./apps.js'),
|
||||
path = require('path'),
|
||||
{ Readable } = require('stream'),
|
||||
safe = require('safetydance'),
|
||||
services = require('./services.js'),
|
||||
superagent = require('./superagent.js');
|
||||
|
||||
function translateContainerStatsSync(stats) {
|
||||
@@ -361,7 +356,7 @@ async function readSystemFromGraphite(options) {
|
||||
// Disk:
|
||||
// writing: fio --name=rate-test --filename=tempfile --rw=write --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
||||
// reading: fio --name=rate-test --filename=tempfile --rw=read --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
||||
async function getSystem(options) {
|
||||
async function get(options) {
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
|
||||
const result = {};
|
||||
@@ -379,74 +374,18 @@ async function getSystem(options) {
|
||||
return result;
|
||||
}
|
||||
|
||||
async function getSystemStream(options) {
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
|
||||
const INTERVAL_MSECS = 1000;
|
||||
let intervalId = null, oldMetrics = null;
|
||||
|
||||
const metricsStream = new Readable({
|
||||
objectMode: true,
|
||||
read(/*size*/) { /* ignored, we push via interval */ },
|
||||
destroy(error, callback) {
|
||||
clearInterval(intervalId);
|
||||
callback(error);
|
||||
}
|
||||
});
|
||||
|
||||
intervalId = setInterval(async () => {
|
||||
const [error, metrics] = await safe(readSystemMetrics());
|
||||
if (error) return metricsStream.destroy(error);
|
||||
|
||||
const cpuPercent = oldMetrics ? (metrics.userMsecs + metrics.sysMsecs - oldMetrics.userMsecs - oldMetrics.sysMsecs) * 100 / INTERVAL_MSECS : null;
|
||||
const blockReadRate = oldMetrics ? (metrics.blockRead - oldMetrics.blockRead) / (INTERVAL_MSECS/1000) : null;
|
||||
const blockWriteRate = oldMetrics ? (metrics.blockWrite - oldMetrics.blockWrite) / (INTERVAL_MSECS/1000) : null;
|
||||
const networkReadRate = oldMetrics ? (metrics.networkRead - oldMetrics.networkRead) / (INTERVAL_MSECS/1000) : null;
|
||||
const networkWriteRate = oldMetrics ? (metrics.networkWrite - oldMetrics.networkWrite) / (INTERVAL_MSECS/1000) : null;
|
||||
|
||||
oldMetrics = metrics;
|
||||
|
||||
const nowSecs = Date.now() / 1000; // to match graphite return value
|
||||
metricsStream.push({
|
||||
cpu: [ cpuPercent, nowSecs ],
|
||||
memory: [ metrics.memoryUsed, nowSecs ],
|
||||
swap: [ metrics.swapUsed, nowSecs ],
|
||||
|
||||
blockReadRate: [ blockReadRate, nowSecs ],
|
||||
blockWriteRate: [ blockWriteRate, nowSecs ],
|
||||
blockReadTotal: metrics.blockRead,
|
||||
blockWriteTotal: metrics.blockWrite,
|
||||
|
||||
networkReadRate: [ networkReadRate, nowSecs ],
|
||||
networkWriteRate: [ networkWriteRate, nowSecs ],
|
||||
networkReadTotal: metrics.networkRead,
|
||||
networkWriteTotal: metrics.networkWrite,
|
||||
});
|
||||
}, INTERVAL_MSECS);
|
||||
|
||||
return metricsStream;
|
||||
}
|
||||
|
||||
async function getContainerStream(name, options) {
|
||||
async function pipeContainerMetrics(name, metricsStream, ac) {
|
||||
assert.strictEqual(typeof name, 'string');
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
assert.strictEqual(typeof metricsStream, 'object');
|
||||
assert.strictEqual(typeof ac, 'object');
|
||||
|
||||
let oldMetrics = null;
|
||||
|
||||
const metricsStream = new Readable({
|
||||
objectMode: true,
|
||||
read(/*size*/) { /* ignored, we push via interval */ },
|
||||
destroy(error, callback) {
|
||||
statsStream.destroy(); // double destroy is a no-op
|
||||
callback(error);
|
||||
}
|
||||
});
|
||||
|
||||
// we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups
|
||||
const [error, statsStream] = await safe(docker.getStats(name, { stream: true }));
|
||||
if (error) throw new Error(`Container stopped or missing: ${error.message}`);
|
||||
if (error) return; // container stopped or missing, silently ignore
|
||||
|
||||
statsStream.on('error', (error) => metricsStream.destroy(error)); // double destroy is a no-op
|
||||
statsStream.on('error', (error) => debug(error));
|
||||
statsStream.on('data', (data) => {
|
||||
const stats = JSON.parse(data.toString('utf8'));
|
||||
const metrics = translateContainerStatsSync(stats);
|
||||
@@ -465,7 +404,8 @@ async function getContainerStream(name, options) {
|
||||
oldMetrics = metrics;
|
||||
|
||||
const nowSecs = ts.getTime() / 1000; // conver to secs to match graphite return value
|
||||
metricsStream.push({
|
||||
const result = {};
|
||||
result[name] = {
|
||||
cpu: [ cpuPercent, nowSecs ],
|
||||
memory: [ memoryUsed, nowSecs ],
|
||||
|
||||
@@ -478,8 +418,80 @@ async function getContainerStream(name, options) {
|
||||
networkWriteRate: [ networkWriteRate, nowSecs ],
|
||||
networkReadTotal: metrics.networkRead,
|
||||
networkWriteTotal: metrics.networkWrite,
|
||||
});
|
||||
};
|
||||
metricsStream.push(result);
|
||||
});
|
||||
|
||||
ac.signal.addEventListener('abort', () => { // there is event.type and ac.signal.reason
|
||||
statsStream.destroy(ac.signal.reason);
|
||||
}, { once: true });
|
||||
}
|
||||
|
||||
async function pipeSystemMetrics(metricsStream, ac) {
|
||||
assert.strictEqual(typeof metricsStream, 'object');
|
||||
assert.strictEqual(typeof ac, 'object');
|
||||
|
||||
const INTERVAL_MSECS = 1000;
|
||||
let oldMetrics = null;
|
||||
|
||||
const intervalId = setInterval(async () => {
|
||||
const [error, metrics] = await safe(readSystemMetrics());
|
||||
if (error) return metricsStream.destroy(error);
|
||||
|
||||
const cpuPercent = oldMetrics ? (metrics.userMsecs + metrics.sysMsecs - oldMetrics.userMsecs - oldMetrics.sysMsecs) * 100 / INTERVAL_MSECS : null;
|
||||
const blockReadRate = oldMetrics ? (metrics.blockRead - oldMetrics.blockRead) / (INTERVAL_MSECS/1000) : null;
|
||||
const blockWriteRate = oldMetrics ? (metrics.blockWrite - oldMetrics.blockWrite) / (INTERVAL_MSECS/1000) : null;
|
||||
const networkReadRate = oldMetrics ? (metrics.networkRead - oldMetrics.networkRead) / (INTERVAL_MSECS/1000) : null;
|
||||
const networkWriteRate = oldMetrics ? (metrics.networkWrite - oldMetrics.networkWrite) / (INTERVAL_MSECS/1000) : null;
|
||||
|
||||
oldMetrics = metrics;
|
||||
|
||||
const nowSecs = Date.now() / 1000; // to match graphite return value
|
||||
const systemStats = {
|
||||
cpu: [ cpuPercent, nowSecs ],
|
||||
memory: [ metrics.memoryUsed, nowSecs ],
|
||||
swap: [ metrics.swapUsed, nowSecs ],
|
||||
|
||||
blockReadRate: [ blockReadRate, nowSecs ],
|
||||
blockWriteRate: [ blockWriteRate, nowSecs ],
|
||||
blockReadTotal: metrics.blockRead,
|
||||
blockWriteTotal: metrics.blockWrite,
|
||||
|
||||
networkReadRate: [ networkReadRate, nowSecs ],
|
||||
networkWriteRate: [ networkWriteRate, nowSecs ],
|
||||
networkReadTotal: metrics.networkRead,
|
||||
networkWriteTotal: metrics.networkWrite,
|
||||
};
|
||||
metricsStream.push({ system: systemStats });
|
||||
}, INTERVAL_MSECS);
|
||||
|
||||
ac.signal.addEventListener('abort', () => { // there is event.type and ac.signal.reason
|
||||
clearInterval(intervalId);
|
||||
}, { once: true });
|
||||
}
|
||||
|
||||
async function getStream(options) {
|
||||
assert.strictEqual(typeof options, 'object');
|
||||
|
||||
const ac = new AbortController();
|
||||
|
||||
const metricsStream = new Readable({
|
||||
objectMode: true,
|
||||
read(/*size*/) { /* ignored, we push via interval */ },
|
||||
destroy(error, callback) {
|
||||
ac.abort(error);
|
||||
callback(error);
|
||||
}
|
||||
});
|
||||
|
||||
if (options.system) pipeSystemMetrics(metricsStream, ac);
|
||||
for (const appId of options.appIds) {
|
||||
pipeContainerMetrics(appId, metricsStream, ac);
|
||||
}
|
||||
|
||||
for (const serviceId of options.serviceIds) {
|
||||
pipeContainerMetrics(serviceId, metricsStream, ac);
|
||||
}
|
||||
|
||||
return metricsStream;
|
||||
}
|
||||
|
||||
+2
-2
@@ -1076,7 +1076,7 @@ async function getMetrics(req, res, next) {
|
||||
const fromSecs = parseInt(req.query.fromSecs);
|
||||
const intervalSecs = parseInt(req.query.intervalSecs);
|
||||
const noNullPoints = !!req.query.noNullPoints;
|
||||
const [error, result] = await safe(metrics.getContainer(req.resources.app.id, { fromSecs, noNullPoints, intervalSecs }));
|
||||
const [error, result] = await safe(metrics.get({ fromSecs, noNullPoints, intervalSecs, appIds: [req.resources.app.id] }));
|
||||
if (error) return next(new HttpError(500, error));
|
||||
|
||||
next(new HttpSuccess(200, result));
|
||||
@@ -1085,7 +1085,7 @@ async function getMetrics(req, res, next) {
|
||||
async function getMetricStream(req, res, next) {
|
||||
if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
|
||||
|
||||
const [error, metricStream] = await safe(metrics.getContainerStream(req.resources.app.id, {}));
|
||||
const [error, metricStream] = await safe(metrics.getStream(req.resources.app.id, { appIds: [req.resources.app.id] }));
|
||||
if (error) return next(BoxError.toHttpError(error));
|
||||
|
||||
res.writeHead(200, {
|
||||
|
||||
@@ -145,7 +145,8 @@ async function getMetrics(req, res, next) {
|
||||
const fromSecs = parseInt(req.query.fromSecs);
|
||||
const intervalSecs = parseInt(req.query.intervalSecs);
|
||||
const noNullPoints = !!req.query.noNullPoints;
|
||||
const [error, result] = await safe(metrics.getContainer(req.params.service, { fromSecs, intervalSecs, noNullPoints }));
|
||||
|
||||
const [error, result] = await safe(metrics.get(req.params.service, { fromSecs, intervalSecs, noNullPoints, serviceIds: [req.params.service] }));
|
||||
if (error) return next(new HttpError(500, error));
|
||||
|
||||
next(new HttpSuccess(200, result));
|
||||
|
||||
@@ -129,7 +129,7 @@ async function getMetrics(req, res, next) {
|
||||
const appIds = 'appId' in req.query ? (Array.isArray(req.query.appId) ? req.query.appId : [ req.query.appId ]) : [];
|
||||
const serviceIds = 'serviceId' in req.query ? (Array.isArray(req.query.serviceId) ? req.query.serviceId : [ req.query.serviceId ]) : [];
|
||||
|
||||
const [error, result] = await safe(metrics.getSystem({ fromSecs, intervalSecs, noNullPoints, system, appIds, serviceIds }));
|
||||
const [error, result] = await safe(metrics.get({ fromSecs, intervalSecs, noNullPoints, system, appIds, serviceIds }));
|
||||
if (error) return next(new HttpError(500, error));
|
||||
|
||||
next(new HttpSuccess(200, result));
|
||||
@@ -138,7 +138,11 @@ async function getMetrics(req, res, next) {
|
||||
async function getMetricStream(req, res, next) {
|
||||
if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
|
||||
|
||||
const [error, metricStream] = await safe(metrics.getSystemStream({}));
|
||||
const system = req.query.system === 'true';
|
||||
const appIds = 'appId' in req.query ? (Array.isArray(req.query.appId) ? req.query.appId : [ req.query.appId ]) : [];
|
||||
const serviceIds = 'serviceId' in req.query ? (Array.isArray(req.query.serviceId) ? req.query.serviceId : [ req.query.serviceId ]) : [];
|
||||
|
||||
const [error, metricStream] = await safe(metrics.getStream({ system, appIds, serviceIds }));
|
||||
if (error) return next(BoxError.toHttpError(error));
|
||||
|
||||
res.writeHead(200, {
|
||||
|
||||
Reference in New Issue
Block a user