metrics: overlay app metrics over system metrics
This commit is contained in:
@@ -21,21 +21,13 @@ const props = defineProps({
|
|||||||
title: String,
|
title: String,
|
||||||
subtext: String,
|
subtext: String,
|
||||||
period: Object, // { hours, format, tooltpFormat }
|
period: Object, // { hours, format, tooltpFormat }
|
||||||
datasetLabels: {
|
|
||||||
type: Array,
|
|
||||||
validator: (val) => Array.isArray(val) && val.every(item => typeof item === 'string')
|
|
||||||
},
|
|
||||||
datasetColors: {
|
|
||||||
type: Array,
|
|
||||||
validator: (val) => Array.isArray(val) && val.every(item => typeof item === 'string')
|
|
||||||
},
|
|
||||||
yscale: String, // cpu, memory
|
yscale: String, // cpu, memory
|
||||||
memory: Number,
|
memory: Number,
|
||||||
cpuCount: Number,
|
cpuCount: Number,
|
||||||
highMark: Number,
|
highMark: Number,
|
||||||
});
|
});
|
||||||
|
|
||||||
function createGraphOptions({ yscale, period, displayLegend, highMark }) {
|
function createGraphOptions({ yscale, period, highMark }) {
|
||||||
let startTime, endTime, stepSize, count; // x axis configuration values
|
let startTime, endTime, stepSize, count; // x axis configuration values
|
||||||
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
@@ -54,8 +46,8 @@ function createGraphOptions({ yscale, period, displayLegend, highMark }) {
|
|||||||
maintainAspectRatio: false,
|
maintainAspectRatio: false,
|
||||||
plugins: {
|
plugins: {
|
||||||
legend: {
|
legend: {
|
||||||
display: displayLegend,
|
display: false,
|
||||||
position: 'bottom'
|
position: 'bottom' // not used, hidden since color code is shown in tooltip
|
||||||
},
|
},
|
||||||
tooltip: {
|
tooltip: {
|
||||||
callbacks: {
|
callbacks: {
|
||||||
@@ -121,10 +113,23 @@ function transformData(data) {
|
|||||||
return { x, y };
|
return { x, y };
|
||||||
}
|
}
|
||||||
|
|
||||||
function setData(...data) {
|
function setDatasets(datasets) {
|
||||||
for (const [index, items] of data.entries()) {
|
graph.data = { datasets: [] };
|
||||||
graph.data.datasets[index].data = items.map(transformData);
|
|
||||||
|
for (const dataset of datasets) {
|
||||||
|
graph.data.datasets.push({
|
||||||
|
label: dataset.label,
|
||||||
|
data: dataset.data.map(transformData),
|
||||||
|
pointRadius: 0,
|
||||||
|
borderWidth: 1, // https://www.chartjs.org/docs/latest/charts/line.html#line-styling
|
||||||
|
tension: 0.4,
|
||||||
|
showLine: true,
|
||||||
|
fill: true,
|
||||||
|
color: dataset.color,
|
||||||
|
stack: dataset.stack || 'stackgroup', // put them all in same stackgroup
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
graph.update('none');
|
graph.update('none');
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,10 +145,10 @@ function advance() {
|
|||||||
graph.update('none');
|
graph.update('none');
|
||||||
}
|
}
|
||||||
|
|
||||||
function pushData(...data) {
|
function pushData(datasetIndex, ...data) {
|
||||||
for (const [index, item] of data.entries()) {
|
for (const [index, item] of data.entries()) {
|
||||||
graph.data.datasets[index].data.push(transformData(item));
|
graph.data.datasets[datasetIndex+index].data.push(transformData(item));
|
||||||
pruneGraphData(graph.data.datasets[index], graph.options);
|
pruneGraphData(graph.data.datasets[datasetIndex+index], graph.options);
|
||||||
}
|
}
|
||||||
graph.update('none');
|
graph.update('none');
|
||||||
}
|
}
|
||||||
@@ -154,22 +159,6 @@ function onPeriodChanged() {
|
|||||||
liveRefreshIntervalId = null;
|
liveRefreshIntervalId = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = { datasets: [] };
|
|
||||||
|
|
||||||
for (const [index, label] of props.datasetLabels.entries()) {
|
|
||||||
data.datasets.push({
|
|
||||||
label: label,
|
|
||||||
data: [],
|
|
||||||
pointRadius: 0,
|
|
||||||
borderWidth: 1, // https://www.chartjs.org/docs/latest/charts/line.html#line-styling
|
|
||||||
tension: 0.4,
|
|
||||||
showLine: true,
|
|
||||||
fill: true,
|
|
||||||
color: props.datasetColors[index],
|
|
||||||
stack: 'stackgroup' // put them all in same stackgroup
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// CPU and Memory graph have known min/max set and auto-scaling gets disabled
|
// CPU and Memory graph have known min/max set and auto-scaling gets disabled
|
||||||
// Disk and Network graphs auto-scale the y values.
|
// Disk and Network graphs auto-scale the y values.
|
||||||
|
|
||||||
@@ -236,12 +225,11 @@ function onPeriodChanged() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// this sets a min 'x' based on current timestamp. so it has to re-created every time the period changes
|
// this sets a min 'x' based on current timestamp. so it has to re-created every time the period changes
|
||||||
const graphOptions = createGraphOptions({ yscale, period: props.period, displayLegend: props.datasetLabels.length > 1, highMark });
|
const graphOptions = createGraphOptions({ yscale, period: props.period, highMark });
|
||||||
|
|
||||||
if (!graph) {
|
if (!graph) {
|
||||||
graph = new Chart(graphNode.value, { type: 'line', data, options: graphOptions });
|
graph = new Chart(graphNode.value, { type: 'line', data: { datasets: [] }, options: graphOptions });
|
||||||
} else {
|
} else {
|
||||||
graph.data = data;
|
|
||||||
graph.options = graphOptions;
|
graph.options = graphOptions;
|
||||||
graph.update('none');
|
graph.update('none');
|
||||||
}
|
}
|
||||||
@@ -261,7 +249,7 @@ onUnmounted(async function () {
|
|||||||
});
|
});
|
||||||
|
|
||||||
defineExpose({
|
defineExpose({
|
||||||
setData,
|
setDatasets,
|
||||||
pushData,
|
pushData,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -4,14 +4,16 @@ import { useI18n } from 'vue-i18n';
|
|||||||
const i18n = useI18n();
|
const i18n = useI18n();
|
||||||
const t = i18n.t;
|
const t = i18n.t;
|
||||||
|
|
||||||
import { ref, onMounted, onUnmounted, useTemplateRef, nextTick } from 'vue';
|
import { ref, onMounted, onUnmounted, useTemplateRef, nextTick, watch } from 'vue';
|
||||||
import { SingleSelect } from 'pankow';
|
import { SingleSelect, MultiSelect } from 'pankow';
|
||||||
import Section from './Section.vue';
|
import Section from './Section.vue';
|
||||||
import SystemModel from '../models/SystemModel.js';
|
import SystemModel from '../models/SystemModel.js';
|
||||||
import { prettyDecimalSize } from 'pankow/utils';
|
import { prettyDecimalSize } from 'pankow/utils';
|
||||||
import GraphItem from './GraphItem.vue';
|
import GraphItem from './GraphItem.vue';
|
||||||
|
import AppsModel from '../models/AppsModel.js';
|
||||||
|
|
||||||
const systemModel = SystemModel.create();
|
const systemModel = SystemModel.create();
|
||||||
|
const appsModel = AppsModel.create();
|
||||||
|
|
||||||
const periods = [
|
const periods = [
|
||||||
{ hours: 0, label: t('app.graphs.period.live'), tickFormat: 'hh:mm A', tooltipFormat: 'hh:mm:ss A' },
|
{ hours: 0, label: t('app.graphs.period.live'), tickFormat: 'hh:mm A', tooltipFormat: 'hh:mm:ss A' },
|
||||||
@@ -36,50 +38,112 @@ const networkWriteTotal = ref(0);
|
|||||||
const blockReadTotal = ref(0);
|
const blockReadTotal = ref(0);
|
||||||
const blockWriteTotal = ref(0);
|
const blockWriteTotal = ref(0);
|
||||||
|
|
||||||
|
const containers = ref([]);
|
||||||
|
const allContainers = ref([]);
|
||||||
|
|
||||||
let systemMemory = {};
|
let systemMemory = {};
|
||||||
let systemCpus = {};
|
let systemCpus = {};
|
||||||
let metricStream = null;
|
let metricStream = null;
|
||||||
|
|
||||||
async function liveRefresh() {
|
async function liveRefresh() {
|
||||||
metricStream = await systemModel.getMetricStream();
|
const options = {
|
||||||
|
system: true,
|
||||||
|
appIds: containers.value.map(c => c.id),
|
||||||
|
serviceIds: []
|
||||||
|
};
|
||||||
|
metricStream = await systemModel.getMetricStream(options);
|
||||||
metricStream.onerror = (error) => console.log('event stream error:', error);
|
metricStream.onerror = (error) => console.log('event stream error:', error);
|
||||||
metricStream.onmessage = (message) => {
|
metricStream.onmessage = (message) => {
|
||||||
const data = JSON.parse(message.data);
|
const data = JSON.parse(message.data);
|
||||||
|
|
||||||
cpuGraphItem.value.pushData(data.cpu);
|
for (const [id, metric] of Object.entries(data)) {
|
||||||
memoryGraphItem.value.pushData(data.memory, data.swap);
|
const idx = id !== 'system' ? containers.value.findIndex(c => c.id === id) : containers.value.length;
|
||||||
diskGraphItem.value.pushData(data.blockReadRate, data.blockWriteRate);
|
|
||||||
networkGraphItem.value.pushData(data.networkReadRate, data.networkWriteRate);
|
|
||||||
|
|
||||||
blockReadTotal.value = prettyDecimalSize(data.blockReadTotal);
|
cpuGraphItem.value.pushData(idx, metric.cpu);
|
||||||
blockWriteTotal.value = prettyDecimalSize(data.blockWriteTotal);
|
memoryGraphItem.value.pushData(idx*2, metric.memory, metric.swap || []); // apps have no swap
|
||||||
networkReadTotal.value = prettyDecimalSize(data.networkReadTotal);
|
diskGraphItem.value.pushData(idx*2, metric.blockReadRate, metric.blockWriteRate);
|
||||||
networkWriteTotal.value = prettyDecimalSize(data.networkWriteTotal);
|
networkGraphItem.value.pushData(idx*2, metric.networkReadRate, metric.networkWriteRate);
|
||||||
|
|
||||||
|
if (id === 'system') {
|
||||||
|
blockReadTotal.value = prettyDecimalSize(metric.blockReadTotal);
|
||||||
|
blockWriteTotal.value = prettyDecimalSize(metric.blockWriteTotal);
|
||||||
|
networkReadTotal.value = prettyDecimalSize(metric.networkReadTotal);
|
||||||
|
networkWriteTotal.value = prettyDecimalSize(metric.networkWriteTotal);
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function onPeriodChange() {
|
function generateConsistentColors(n, saturation = 90, lightness = 90) {
|
||||||
|
const baseHue = 204; // from #9ad0f5 → hsl(204,82%,78%)
|
||||||
|
const colors = [];
|
||||||
|
const step = 360 / n;
|
||||||
|
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
const hue = Math.round((baseHue + step * i) % 360); // rotate hue, wrap at 360
|
||||||
|
colors.push(`hsl(${hue}, ${saturation}%, ${lightness}%)`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return colors;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createDatasets() {
|
||||||
|
const colors = generateConsistentColors((containers.value.length+1)*2); // 1 for the 'system'
|
||||||
|
|
||||||
|
const datasets = {
|
||||||
|
cpu: [],
|
||||||
|
memory: [],
|
||||||
|
disk: [],
|
||||||
|
network: [],
|
||||||
|
};
|
||||||
|
const appIds = containers.value.map(c => c.id);
|
||||||
|
for (const [idx, id] of appIds.concat(['system']).entries()) { // live stream code depends on this concat order!
|
||||||
|
const prefix = id === 'system' ? 'System' : containers.value[idx].label;
|
||||||
|
|
||||||
|
datasets.cpu.push({ label: `${prefix} CPU`, color: colors[idx*2], stack: `${prefix}-cpu`, data: [] });
|
||||||
|
datasets.memory.push({ label: `${prefix} Memory`, color: colors[idx*2], stack: `${prefix}-memswap`, data: [] });
|
||||||
|
datasets.memory.push({ label: `${prefix} Swap`, color: colors[idx*2 + 1], stack: `${prefix}-memswap`, data: [] });
|
||||||
|
|
||||||
|
datasets.disk.push({ label: `${prefix} Read`, color: colors[idx*2], stack: `${prefix}-read`, data: [] });
|
||||||
|
datasets.disk.push({ label: `${prefix} Write`, color: colors[idx*2 + 1], stack: `${prefix}-write`, prefix, data: [] });
|
||||||
|
|
||||||
|
datasets.network.push({ label: `${prefix} RX`, color: colors[idx*2], stack: `${prefix}-rx`, data: [] });
|
||||||
|
datasets.network.push({ label: `${prefix} TX`, color: colors[idx*2 + 1], stack: `${prefix}-tx`, data: [] });
|
||||||
|
}
|
||||||
|
|
||||||
|
return datasets;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rebuild() {
|
||||||
if (metricStream) {
|
if (metricStream) {
|
||||||
metricStream.close();
|
metricStream.close();
|
||||||
metricStream = null;
|
metricStream = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (period.value.hours === 0) return await liveRefresh();
|
const datasets = createDatasets();
|
||||||
|
|
||||||
|
if (period.value.hours !== 0) {
|
||||||
const options = {
|
const options = {
|
||||||
fromSecs: period.value.hours * 60 * 60,
|
fromSecs: period.value.hours * 60 * 60,
|
||||||
intervalSecs: period.value.intervalSecs,
|
intervalSecs: period.value.intervalSecs,
|
||||||
system: true,
|
system: true,
|
||||||
appIds: [],
|
appIds: containers.value.map(c => c.id),
|
||||||
serviceIds: []
|
serviceIds: []
|
||||||
};
|
};
|
||||||
const [error, metrics] = await systemModel.getMetrics(options);
|
const [error, metrics] = await systemModel.getMetrics(options);
|
||||||
if (error) return console.error(error);
|
if (error) return console.error(error);
|
||||||
|
|
||||||
cpuGraphItem.value.setData(metrics.system.cpu);
|
const appIds = containers.value.map(c => c.id);
|
||||||
memoryGraphItem.value.setData(metrics.system.memory, metrics.system.swap);
|
for (const [idx, id] of appIds.concat(['system']).entries()) {
|
||||||
diskGraphItem.value.setData(metrics.system.blockReadRate, metrics.system.blockWriteRate);
|
if (!metrics[id]) continue;
|
||||||
networkGraphItem.value.setData(metrics.system.networkReadRate, metrics.system.networkWriteRate);
|
datasets.cpu[idx].data = metrics[id].cpu;
|
||||||
|
datasets.memory[idx*2].data = metrics[id].memory;
|
||||||
|
datasets.memory[idx*2 + 1].data = metrics[id].swap || []; // apps have no swap
|
||||||
|
datasets.disk[idx*2].data = metrics[id].blockReadRate;
|
||||||
|
datasets.disk[idx*2 + 1].data = metrics[id].blockWriteRate;
|
||||||
|
datasets.network[idx*2].data = metrics[id].networkReadRate;
|
||||||
|
datasets.network[idx*2 + 1].data = metrics[id].networkWriteRate;
|
||||||
|
}
|
||||||
|
|
||||||
networkReadTotal.value = prettyDecimalSize(metrics.system.networkReadTotal);
|
networkReadTotal.value = prettyDecimalSize(metrics.system.networkReadTotal);
|
||||||
networkWriteTotal.value = prettyDecimalSize(metrics.system.networkWriteTotal);
|
networkWriteTotal.value = prettyDecimalSize(metrics.system.networkWriteTotal);
|
||||||
@@ -87,22 +151,35 @@ async function onPeriodChange() {
|
|||||||
blockWriteTotal.value = prettyDecimalSize(metrics.system.blockWriteTotal);
|
blockWriteTotal.value = prettyDecimalSize(metrics.system.blockWriteTotal);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cpuGraphItem.value.setDatasets(datasets.cpu);
|
||||||
|
memoryGraphItem.value.setDatasets(datasets.memory);
|
||||||
|
diskGraphItem.value.setDatasets(datasets.disk);
|
||||||
|
networkGraphItem.value.setDatasets(datasets.network);
|
||||||
|
|
||||||
|
if (period.value.hours === 0) return await liveRefresh();
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
let error, result;
|
let error, result;
|
||||||
|
|
||||||
[error, result] = await systemModel.memory();
|
[error, result] = await systemModel.memory();
|
||||||
if (error) return console.error(error);
|
if (error) return console.error(error);
|
||||||
|
|
||||||
systemMemory = result;
|
systemMemory = result;
|
||||||
|
|
||||||
[error, result] = await systemModel.cpus();
|
[error, result] = await systemModel.cpus();
|
||||||
if (error) return console.error(error);
|
if (error) return console.error(error);
|
||||||
|
|
||||||
systemCpus = result;
|
systemCpus = result;
|
||||||
|
|
||||||
|
[error, result] = await appsModel.list();
|
||||||
|
if (error) return console.error(error);
|
||||||
|
result.forEach(a => a.label = (a.label || a.fqdn));
|
||||||
|
allContainers.value = result;
|
||||||
|
containers.value = [];
|
||||||
|
|
||||||
busy.value = false;
|
busy.value = false;
|
||||||
await nextTick();
|
await nextTick();
|
||||||
|
|
||||||
await onPeriodChange();
|
await rebuild();
|
||||||
});
|
});
|
||||||
|
|
||||||
onUnmounted(async () => {
|
onUnmounted(async () => {
|
||||||
@@ -114,7 +191,8 @@ onUnmounted(async () => {
|
|||||||
<template>
|
<template>
|
||||||
<Section :title="$t('system.graphs.title')">
|
<Section :title="$t('system.graphs.title')">
|
||||||
<template #header-buttons>
|
<template #header-buttons>
|
||||||
<SingleSelect @select="onPeriodChange()" v-model="period" :options="periods" option-label="label"/>
|
<MultiSelect @select="rebuild()" v-model="containers" :options="allContainers" option-label="label" :search-threshold="20"/>
|
||||||
|
<SingleSelect @select="rebuild()" v-model="period" :options="periods" option-label="label"/>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<div class="graphs" v-if="!busy">
|
<div class="graphs" v-if="!busy">
|
||||||
@@ -123,8 +201,6 @@ onUnmounted(async () => {
|
|||||||
:subtext='systemCpus.length ? `${systemCpus.length} Core "${systemCpus[0].model}"` : ""'
|
:subtext='systemCpus.length ? `${systemCpus.length} Core "${systemCpus[0].model}"` : ""'
|
||||||
:period="period"
|
:period="period"
|
||||||
yscale="cpu"
|
yscale="cpu"
|
||||||
:dataset-labels="['CPU']"
|
|
||||||
:dataset-colors="['#9ad0f5']"
|
|
||||||
:cpu-count="systemCpus.length"
|
:cpu-count="systemCpus.length"
|
||||||
>
|
>
|
||||||
</GraphItem>
|
</GraphItem>
|
||||||
@@ -134,8 +210,6 @@ onUnmounted(async () => {
|
|||||||
:subtext="`RAM: ${prettyDecimalSize(systemMemory.memory)} Swap: ${prettyDecimalSize(systemMemory.swap)}`"
|
:subtext="`RAM: ${prettyDecimalSize(systemMemory.memory)} Swap: ${prettyDecimalSize(systemMemory.swap)}`"
|
||||||
:period="period"
|
:period="period"
|
||||||
yscale="memory"
|
yscale="memory"
|
||||||
:dataset-labels="['Memory', 'Swap']"
|
|
||||||
:dataset-colors="['#9ad0f5', '#ffb1c1']"
|
|
||||||
:memory="systemMemory.memory + systemMemory.swap"
|
:memory="systemMemory.memory + systemMemory.swap"
|
||||||
>
|
>
|
||||||
</GraphItem>
|
</GraphItem>
|
||||||
@@ -145,8 +219,6 @@ onUnmounted(async () => {
|
|||||||
:subtext="$t('app.graphs.diskIOTotal', { read: blockReadTotal, write: blockWriteTotal })"
|
:subtext="$t('app.graphs.diskIOTotal', { read: blockReadTotal, write: blockWriteTotal })"
|
||||||
:period="period"
|
:period="period"
|
||||||
yscale="disk"
|
yscale="disk"
|
||||||
:dataset-labels="['Read', 'Write']"
|
|
||||||
:dataset-colors="['#9ad0f5', '#ffb1c1']"
|
|
||||||
>
|
>
|
||||||
</GraphItem>
|
</GraphItem>
|
||||||
|
|
||||||
@@ -155,8 +227,6 @@ onUnmounted(async () => {
|
|||||||
:subtext="$t('app.graphs.networkIOTotal', { inbound: networkReadTotal, outbound: networkWriteTotal })"
|
:subtext="$t('app.graphs.networkIOTotal', { inbound: networkReadTotal, outbound: networkWriteTotal })"
|
||||||
:period="period"
|
:period="period"
|
||||||
yscale="network"
|
yscale="network"
|
||||||
:dataset-labels="['RX', 'TX']"
|
|
||||||
:dataset-colors="['#9ad0f5', '#ffb1c1']"
|
|
||||||
>
|
>
|
||||||
</GraphItem>
|
</GraphItem>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -102,8 +102,16 @@ function create() {
|
|||||||
if (error || result.status !== 200) return [error || result];
|
if (error || result.status !== 200) return [error || result];
|
||||||
return [null, result.body];
|
return [null, result.body];
|
||||||
},
|
},
|
||||||
async getMetricStream() {
|
async getMetricStream(options) {
|
||||||
return new EventSource(`${API_ORIGIN}/api/v1/system/metricstream?access_token=${accessToken}`);
|
const query = [
|
||||||
|
['system', String(!!options.system)],
|
||||||
|
...options.appIds.map(id => ['appId', id]), // multiple appId=xx
|
||||||
|
...options.serviceIds.map(id => ['serviceId', id]), // multiple serviceId=xx
|
||||||
|
['access_token', accessToken]
|
||||||
|
];
|
||||||
|
|
||||||
|
const queryString = new URLSearchParams(query).toString();
|
||||||
|
return new EventSource(`${API_ORIGIN}/api/v1/system/metricstream?${queryString}`);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
+82
-70
@@ -1,12 +1,8 @@
|
|||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
exports = module.exports = {
|
exports = module.exports = {
|
||||||
getSystem,
|
get,
|
||||||
getSystemStream,
|
getStream,
|
||||||
|
|
||||||
getContainer,
|
|
||||||
getContainerStream,
|
|
||||||
|
|
||||||
sendToGraphite
|
sendToGraphite
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -22,7 +18,6 @@ const apps = require('./apps.js'),
|
|||||||
path = require('path'),
|
path = require('path'),
|
||||||
{ Readable } = require('stream'),
|
{ Readable } = require('stream'),
|
||||||
safe = require('safetydance'),
|
safe = require('safetydance'),
|
||||||
services = require('./services.js'),
|
|
||||||
superagent = require('./superagent.js');
|
superagent = require('./superagent.js');
|
||||||
|
|
||||||
function translateContainerStatsSync(stats) {
|
function translateContainerStatsSync(stats) {
|
||||||
@@ -361,7 +356,7 @@ async function readSystemFromGraphite(options) {
|
|||||||
// Disk:
|
// Disk:
|
||||||
// writing: fio --name=rate-test --filename=tempfile --rw=write --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
// writing: fio --name=rate-test --filename=tempfile --rw=write --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
||||||
// reading: fio --name=rate-test --filename=tempfile --rw=read --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
// reading: fio --name=rate-test --filename=tempfile --rw=read --bs=4k --ioengine=libaio --rate=20M --size=5000M --runtime=150 --direct=1. test with iotop
|
||||||
async function getSystem(options) {
|
async function get(options) {
|
||||||
assert.strictEqual(typeof options, 'object');
|
assert.strictEqual(typeof options, 'object');
|
||||||
|
|
||||||
const result = {};
|
const result = {};
|
||||||
@@ -379,74 +374,18 @@ async function getSystem(options) {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getSystemStream(options) {
|
async function pipeContainerMetrics(name, metricsStream, ac) {
|
||||||
assert.strictEqual(typeof options, 'object');
|
|
||||||
|
|
||||||
const INTERVAL_MSECS = 1000;
|
|
||||||
let intervalId = null, oldMetrics = null;
|
|
||||||
|
|
||||||
const metricsStream = new Readable({
|
|
||||||
objectMode: true,
|
|
||||||
read(/*size*/) { /* ignored, we push via interval */ },
|
|
||||||
destroy(error, callback) {
|
|
||||||
clearInterval(intervalId);
|
|
||||||
callback(error);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
intervalId = setInterval(async () => {
|
|
||||||
const [error, metrics] = await safe(readSystemMetrics());
|
|
||||||
if (error) return metricsStream.destroy(error);
|
|
||||||
|
|
||||||
const cpuPercent = oldMetrics ? (metrics.userMsecs + metrics.sysMsecs - oldMetrics.userMsecs - oldMetrics.sysMsecs) * 100 / INTERVAL_MSECS : null;
|
|
||||||
const blockReadRate = oldMetrics ? (metrics.blockRead - oldMetrics.blockRead) / (INTERVAL_MSECS/1000) : null;
|
|
||||||
const blockWriteRate = oldMetrics ? (metrics.blockWrite - oldMetrics.blockWrite) / (INTERVAL_MSECS/1000) : null;
|
|
||||||
const networkReadRate = oldMetrics ? (metrics.networkRead - oldMetrics.networkRead) / (INTERVAL_MSECS/1000) : null;
|
|
||||||
const networkWriteRate = oldMetrics ? (metrics.networkWrite - oldMetrics.networkWrite) / (INTERVAL_MSECS/1000) : null;
|
|
||||||
|
|
||||||
oldMetrics = metrics;
|
|
||||||
|
|
||||||
const nowSecs = Date.now() / 1000; // to match graphite return value
|
|
||||||
metricsStream.push({
|
|
||||||
cpu: [ cpuPercent, nowSecs ],
|
|
||||||
memory: [ metrics.memoryUsed, nowSecs ],
|
|
||||||
swap: [ metrics.swapUsed, nowSecs ],
|
|
||||||
|
|
||||||
blockReadRate: [ blockReadRate, nowSecs ],
|
|
||||||
blockWriteRate: [ blockWriteRate, nowSecs ],
|
|
||||||
blockReadTotal: metrics.blockRead,
|
|
||||||
blockWriteTotal: metrics.blockWrite,
|
|
||||||
|
|
||||||
networkReadRate: [ networkReadRate, nowSecs ],
|
|
||||||
networkWriteRate: [ networkWriteRate, nowSecs ],
|
|
||||||
networkReadTotal: metrics.networkRead,
|
|
||||||
networkWriteTotal: metrics.networkWrite,
|
|
||||||
});
|
|
||||||
}, INTERVAL_MSECS);
|
|
||||||
|
|
||||||
return metricsStream;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getContainerStream(name, options) {
|
|
||||||
assert.strictEqual(typeof name, 'string');
|
assert.strictEqual(typeof name, 'string');
|
||||||
assert.strictEqual(typeof options, 'object');
|
assert.strictEqual(typeof metricsStream, 'object');
|
||||||
|
assert.strictEqual(typeof ac, 'object');
|
||||||
|
|
||||||
let oldMetrics = null;
|
let oldMetrics = null;
|
||||||
|
|
||||||
const metricsStream = new Readable({
|
|
||||||
objectMode: true,
|
|
||||||
read(/*size*/) { /* ignored, we push via interval */ },
|
|
||||||
destroy(error, callback) {
|
|
||||||
statsStream.destroy(); // double destroy is a no-op
|
|
||||||
callback(error);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups
|
// we used to poll before instead of a stream. but docker caches metrics internally and rate logic has to handle dups
|
||||||
const [error, statsStream] = await safe(docker.getStats(name, { stream: true }));
|
const [error, statsStream] = await safe(docker.getStats(name, { stream: true }));
|
||||||
if (error) throw new Error(`Container stopped or missing: ${error.message}`);
|
if (error) return; // container stopped or missing, silently ignore
|
||||||
|
|
||||||
statsStream.on('error', (error) => metricsStream.destroy(error)); // double destroy is a no-op
|
statsStream.on('error', (error) => debug(error));
|
||||||
statsStream.on('data', (data) => {
|
statsStream.on('data', (data) => {
|
||||||
const stats = JSON.parse(data.toString('utf8'));
|
const stats = JSON.parse(data.toString('utf8'));
|
||||||
const metrics = translateContainerStatsSync(stats);
|
const metrics = translateContainerStatsSync(stats);
|
||||||
@@ -465,7 +404,8 @@ async function getContainerStream(name, options) {
|
|||||||
oldMetrics = metrics;
|
oldMetrics = metrics;
|
||||||
|
|
||||||
const nowSecs = ts.getTime() / 1000; // conver to secs to match graphite return value
|
const nowSecs = ts.getTime() / 1000; // conver to secs to match graphite return value
|
||||||
metricsStream.push({
|
const result = {};
|
||||||
|
result[name] = {
|
||||||
cpu: [ cpuPercent, nowSecs ],
|
cpu: [ cpuPercent, nowSecs ],
|
||||||
memory: [ memoryUsed, nowSecs ],
|
memory: [ memoryUsed, nowSecs ],
|
||||||
|
|
||||||
@@ -478,8 +418,80 @@ async function getContainerStream(name, options) {
|
|||||||
networkWriteRate: [ networkWriteRate, nowSecs ],
|
networkWriteRate: [ networkWriteRate, nowSecs ],
|
||||||
networkReadTotal: metrics.networkRead,
|
networkReadTotal: metrics.networkRead,
|
||||||
networkWriteTotal: metrics.networkWrite,
|
networkWriteTotal: metrics.networkWrite,
|
||||||
|
};
|
||||||
|
metricsStream.push(result);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
ac.signal.addEventListener('abort', () => { // there is event.type and ac.signal.reason
|
||||||
|
statsStream.destroy(ac.signal.reason);
|
||||||
|
}, { once: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pipeSystemMetrics(metricsStream, ac) {
|
||||||
|
assert.strictEqual(typeof metricsStream, 'object');
|
||||||
|
assert.strictEqual(typeof ac, 'object');
|
||||||
|
|
||||||
|
const INTERVAL_MSECS = 1000;
|
||||||
|
let oldMetrics = null;
|
||||||
|
|
||||||
|
const intervalId = setInterval(async () => {
|
||||||
|
const [error, metrics] = await safe(readSystemMetrics());
|
||||||
|
if (error) return metricsStream.destroy(error);
|
||||||
|
|
||||||
|
const cpuPercent = oldMetrics ? (metrics.userMsecs + metrics.sysMsecs - oldMetrics.userMsecs - oldMetrics.sysMsecs) * 100 / INTERVAL_MSECS : null;
|
||||||
|
const blockReadRate = oldMetrics ? (metrics.blockRead - oldMetrics.blockRead) / (INTERVAL_MSECS/1000) : null;
|
||||||
|
const blockWriteRate = oldMetrics ? (metrics.blockWrite - oldMetrics.blockWrite) / (INTERVAL_MSECS/1000) : null;
|
||||||
|
const networkReadRate = oldMetrics ? (metrics.networkRead - oldMetrics.networkRead) / (INTERVAL_MSECS/1000) : null;
|
||||||
|
const networkWriteRate = oldMetrics ? (metrics.networkWrite - oldMetrics.networkWrite) / (INTERVAL_MSECS/1000) : null;
|
||||||
|
|
||||||
|
oldMetrics = metrics;
|
||||||
|
|
||||||
|
const nowSecs = Date.now() / 1000; // to match graphite return value
|
||||||
|
const systemStats = {
|
||||||
|
cpu: [ cpuPercent, nowSecs ],
|
||||||
|
memory: [ metrics.memoryUsed, nowSecs ],
|
||||||
|
swap: [ metrics.swapUsed, nowSecs ],
|
||||||
|
|
||||||
|
blockReadRate: [ blockReadRate, nowSecs ],
|
||||||
|
blockWriteRate: [ blockWriteRate, nowSecs ],
|
||||||
|
blockReadTotal: metrics.blockRead,
|
||||||
|
blockWriteTotal: metrics.blockWrite,
|
||||||
|
|
||||||
|
networkReadRate: [ networkReadRate, nowSecs ],
|
||||||
|
networkWriteRate: [ networkWriteRate, nowSecs ],
|
||||||
|
networkReadTotal: metrics.networkRead,
|
||||||
|
networkWriteTotal: metrics.networkWrite,
|
||||||
|
};
|
||||||
|
metricsStream.push({ system: systemStats });
|
||||||
|
}, INTERVAL_MSECS);
|
||||||
|
|
||||||
|
ac.signal.addEventListener('abort', () => { // there is event.type and ac.signal.reason
|
||||||
|
clearInterval(intervalId);
|
||||||
|
}, { once: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getStream(options) {
|
||||||
|
assert.strictEqual(typeof options, 'object');
|
||||||
|
|
||||||
|
const ac = new AbortController();
|
||||||
|
|
||||||
|
const metricsStream = new Readable({
|
||||||
|
objectMode: true,
|
||||||
|
read(/*size*/) { /* ignored, we push via interval */ },
|
||||||
|
destroy(error, callback) {
|
||||||
|
ac.abort(error);
|
||||||
|
callback(error);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (options.system) pipeSystemMetrics(metricsStream, ac);
|
||||||
|
for (const appId of options.appIds) {
|
||||||
|
pipeContainerMetrics(appId, metricsStream, ac);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const serviceId of options.serviceIds) {
|
||||||
|
pipeContainerMetrics(serviceId, metricsStream, ac);
|
||||||
|
}
|
||||||
|
|
||||||
return metricsStream;
|
return metricsStream;
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -1076,7 +1076,7 @@ async function getMetrics(req, res, next) {
|
|||||||
const fromSecs = parseInt(req.query.fromSecs);
|
const fromSecs = parseInt(req.query.fromSecs);
|
||||||
const intervalSecs = parseInt(req.query.intervalSecs);
|
const intervalSecs = parseInt(req.query.intervalSecs);
|
||||||
const noNullPoints = !!req.query.noNullPoints;
|
const noNullPoints = !!req.query.noNullPoints;
|
||||||
const [error, result] = await safe(metrics.getContainer(req.resources.app.id, { fromSecs, noNullPoints, intervalSecs }));
|
const [error, result] = await safe(metrics.get({ fromSecs, noNullPoints, intervalSecs, appIds: [req.resources.app.id] }));
|
||||||
if (error) return next(new HttpError(500, error));
|
if (error) return next(new HttpError(500, error));
|
||||||
|
|
||||||
next(new HttpSuccess(200, result));
|
next(new HttpSuccess(200, result));
|
||||||
@@ -1085,7 +1085,7 @@ async function getMetrics(req, res, next) {
|
|||||||
async function getMetricStream(req, res, next) {
|
async function getMetricStream(req, res, next) {
|
||||||
if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
|
if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
|
||||||
|
|
||||||
const [error, metricStream] = await safe(metrics.getContainerStream(req.resources.app.id, {}));
|
const [error, metricStream] = await safe(metrics.getStream(req.resources.app.id, { appIds: [req.resources.app.id] }));
|
||||||
if (error) return next(BoxError.toHttpError(error));
|
if (error) return next(BoxError.toHttpError(error));
|
||||||
|
|
||||||
res.writeHead(200, {
|
res.writeHead(200, {
|
||||||
|
|||||||
@@ -145,7 +145,8 @@ async function getMetrics(req, res, next) {
|
|||||||
const fromSecs = parseInt(req.query.fromSecs);
|
const fromSecs = parseInt(req.query.fromSecs);
|
||||||
const intervalSecs = parseInt(req.query.intervalSecs);
|
const intervalSecs = parseInt(req.query.intervalSecs);
|
||||||
const noNullPoints = !!req.query.noNullPoints;
|
const noNullPoints = !!req.query.noNullPoints;
|
||||||
const [error, result] = await safe(metrics.getContainer(req.params.service, { fromSecs, intervalSecs, noNullPoints }));
|
|
||||||
|
const [error, result] = await safe(metrics.get(req.params.service, { fromSecs, intervalSecs, noNullPoints, serviceIds: [req.params.service] }));
|
||||||
if (error) return next(new HttpError(500, error));
|
if (error) return next(new HttpError(500, error));
|
||||||
|
|
||||||
next(new HttpSuccess(200, result));
|
next(new HttpSuccess(200, result));
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ async function getMetrics(req, res, next) {
|
|||||||
const appIds = 'appId' in req.query ? (Array.isArray(req.query.appId) ? req.query.appId : [ req.query.appId ]) : [];
|
const appIds = 'appId' in req.query ? (Array.isArray(req.query.appId) ? req.query.appId : [ req.query.appId ]) : [];
|
||||||
const serviceIds = 'serviceId' in req.query ? (Array.isArray(req.query.serviceId) ? req.query.serviceId : [ req.query.serviceId ]) : [];
|
const serviceIds = 'serviceId' in req.query ? (Array.isArray(req.query.serviceId) ? req.query.serviceId : [ req.query.serviceId ]) : [];
|
||||||
|
|
||||||
const [error, result] = await safe(metrics.getSystem({ fromSecs, intervalSecs, noNullPoints, system, appIds, serviceIds }));
|
const [error, result] = await safe(metrics.get({ fromSecs, intervalSecs, noNullPoints, system, appIds, serviceIds }));
|
||||||
if (error) return next(new HttpError(500, error));
|
if (error) return next(new HttpError(500, error));
|
||||||
|
|
||||||
next(new HttpSuccess(200, result));
|
next(new HttpSuccess(200, result));
|
||||||
@@ -138,7 +138,11 @@ async function getMetrics(req, res, next) {
|
|||||||
async function getMetricStream(req, res, next) {
|
async function getMetricStream(req, res, next) {
|
||||||
if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
|
if (req.headers.accept !== 'text/event-stream') return next(new HttpError(400, 'This API call requires EventStream'));
|
||||||
|
|
||||||
const [error, metricStream] = await safe(metrics.getSystemStream({}));
|
const system = req.query.system === 'true';
|
||||||
|
const appIds = 'appId' in req.query ? (Array.isArray(req.query.appId) ? req.query.appId : [ req.query.appId ]) : [];
|
||||||
|
const serviceIds = 'serviceId' in req.query ? (Array.isArray(req.query.serviceId) ? req.query.serviceId : [ req.query.serviceId ]) : [];
|
||||||
|
|
||||||
|
const [error, metricStream] = await safe(metrics.getStream({ system, appIds, serviceIds }));
|
||||||
if (error) return next(BoxError.toHttpError(error));
|
if (error) return next(BoxError.toHttpError(error));
|
||||||
|
|
||||||
res.writeHead(200, {
|
res.writeHead(200, {
|
||||||
|
|||||||
Reference in New Issue
Block a user