From af3f9ab1e889c65840976ce5850a9103a65a9dca Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 13:07:00 +0200 Subject: [PATCH 1/7] implement Prometheus client and metrics --- packages/brain/src/index.ts | 11 +- .../brain/src/modules/apiClients/index.ts | 1 + .../modules/apiClients/prometheus/error.ts | 8 + .../modules/apiClients/prometheus/index.ts | 166 ++++++++++++++++++ .../modules/apiClients/prometheus/types.ts | 8 + .../brain/src/modules/apiClients/standard.ts | 4 +- ...fetchAndInsertValidatorsPerformanceData.ts | 5 +- .../cron/trackValidatorsPerformance/index.ts | 5 +- .../sendValidatorsPerformanceNotifications.ts | 29 ++- .../apiClients/prometheus.unit.test.ts | 17 ++ 10 files changed, 246 insertions(+), 8 deletions(-) create mode 100644 packages/brain/src/modules/apiClients/prometheus/error.ts create mode 100644 packages/brain/src/modules/apiClients/prometheus/index.ts create mode 100644 packages/brain/src/modules/apiClients/prometheus/types.ts create mode 100644 packages/brain/test/unit/modules/apiClients/prometheus.unit.test.ts diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts index 3968d9db..c853ff05 100644 --- a/packages/brain/src/index.ts +++ b/packages/brain/src/index.ts @@ -7,7 +7,8 @@ import { BlockExplorerApi, ValidatorApi, DappnodeSignatureVerifier, - DappmanagerApi + DappmanagerApi, + PrometheusApi } from "./modules/apiClients/index.js"; import { startUiServer, startLaunchpadApi } from "./modules/apiServers/index.js"; import * as dotenv from "dotenv"; @@ -57,6 +58,13 @@ logger.debug( ); // Create API instances. Must preceed db initialization +export const prometheusApi = new PrometheusApi({ + baseUrl: executionClientUrl, + minGenesisTime, + secondsPerSlot, + slotsPerEpoch, + network +}); export const signerApi = new Web3SignerApi( { baseUrl: signerUrl, @@ -114,6 +122,7 @@ export const trackValidatorsPerformanceCronTask = new CronJob( beaconchainApi, executionClient, consensusClient, + prometheusApi, dappmanagerApi, sendNotification: true }); diff --git a/packages/brain/src/modules/apiClients/index.ts b/packages/brain/src/modules/apiClients/index.ts index 752c626c..40ac0182 100644 --- a/packages/brain/src/modules/apiClients/index.ts +++ b/packages/brain/src/modules/apiClients/index.ts @@ -1,5 +1,6 @@ export { BlockExplorerApi } from "./blockExplorer/index.js"; export { DappmanagerApi } from "./dappmanager/index.js"; +export { PrometheusApi } from "./prometheus/index.js"; export { BeaconchainApi } from "./beaconchain/index.js"; export { ValidatorApi } from "./validator/index.js"; export { StandardApi } from "./standard.js"; diff --git a/packages/brain/src/modules/apiClients/prometheus/error.ts b/packages/brain/src/modules/apiClients/prometheus/error.ts new file mode 100644 index 00000000..0e93e6a6 --- /dev/null +++ b/packages/brain/src/modules/apiClients/prometheus/error.ts @@ -0,0 +1,8 @@ +import { ApiError } from "../error.js"; + +export class PrometheusApiError extends ApiError { + constructor(message: string) { + super(message); + this.name = "PrometheusApiError"; + } +} diff --git a/packages/brain/src/modules/apiClients/prometheus/index.ts b/packages/brain/src/modules/apiClients/prometheus/index.ts new file mode 100644 index 00000000..c196be75 --- /dev/null +++ b/packages/brain/src/modules/apiClients/prometheus/index.ts @@ -0,0 +1,166 @@ +import { Network } from "@stakingbrain/common"; +import logger from "../../logger/index.js"; +import { StandardApi } from "../standard.js"; +import { AvgHostMetrics } from "./types.js"; + +export class PrometheusApi extends StandardApi { + private readonly minGenesisTime: number; + private readonly slotsPerEpoch: number; + private readonly secondsPerSlot: number; + + constructor({ + baseUrl, + minGenesisTime, + slotsPerEpoch, + secondsPerSlot, + network + }: { + baseUrl: string; + minGenesisTime: number; + slotsPerEpoch: number; + secondsPerSlot: number; + network: Network; + }) { + super({ baseUrl }, network); + this.minGenesisTime = minGenesisTime; + this.slotsPerEpoch = slotsPerEpoch; + this.secondsPerSlot = secondsPerSlot; + } + + /** + * Get average host metrics for a given epoch: + * - avgCpuTemperature + * - avgCpuUsage + * - avgMemoryUsage + * - ioUtilizationPerDisk + */ + public async getPrometheusMetrics({ epoch }: { epoch: number }): Promise { + try { + const { startTimestamp, endTimestamp } = this.calculateEpochTimestamps(epoch); + + // Looks like query_range does not allow to request multiple queries in a single reques + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ioDisk: { metric: { device: string }; values: any[] }[] = await this.getPrometheusDataResult({ + query: `irate(node_disk_io_time_seconds_total{instance="node-exporter.dms.dappnode:9100", job="nodeexporter"}[${endTimestamp - startTimestamp}s])`, + startTimestamp, + endTimestamp + }); + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ioUtilizationPerDisk = ioDisk.reduce((acc: { [key: string]: any }, disk) => { + const device = disk.metric.device; + const utilization = Math.round(parseFloat(disk.values[0][1]) * 100); + acc[device] = utilization; + return acc; + }, {}); + + return { + avgCpuTemperature: await this.getPrometheusAvgMetric({ + query: `avg_over_time(dappmanager_cpu_temperature_celsius{app="dappmanager-custom-metrics", instance="dappmanager.dappnode:80", job="manager_sd", package="dappmanager.dnp.dappnode.eth", service="dappmanager", type="current"}[${endTimestamp - startTimestamp}s])`, + startTimestamp, + endTimestamp + }), + avgCpuUsage: await this.getPrometheusAvgMetric({ + query: `100 * (1 - avg(rate(node_cpu_seconds_total{instance="node-exporter.dms.dappnode:9100", job="nodeexporter", mode="idle"}[${endTimestamp - startTimestamp}s])) by (instance))`, + startTimestamp, + endTimestamp + }), + avgMemoryUsage: await this.getPrometheusAvgMetric({ + query: `100 * (1 - avg(node_memory_MemAvailable_bytes{instance="node-exporter.dms.dappnode:9100", job="nodeexporter"} / node_memory_MemTotal_bytes{instance="node-exporter.dms.dappnode:9100", job="nodeexporter"}) by (instance))`, + startTimestamp, + endTimestamp + }), + ioUtilizationPerDisk + }; + } catch (error) { + logger.error("Failed to get prometheus metrics", error); + throw error; + } + } + + /** + * Query prometheus metric using the endpoint /api/v1/query_range. + * Used to get the data result for later processing. + * + * @see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries + */ + private async getPrometheusDataResult({ + query, + startTimestamp, + endTimestamp + }: { + query: string; + startTimestamp: number; + endTimestamp: number; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }): Promise<{ metric: { device: string }; values: any[] }[]> { + // Construct the request body + const requestBody = new URLSearchParams({ + query, + start: startTimestamp.toString(), + end: endTimestamp.toString(), + step: `10m` // It should be higher than the time range so it returns only one value + }).toString(); + + return ( + await this.request({ + method: "POST", + endpoint: `/api/v1/query_range`, + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: requestBody + }) + ).data.result; + } + + /** + * Query prometheus metric using the endpoint /api/v1/query_range. + * This method assumes there is only 1 metric in the reponse (in the array) + * + * @see https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries + */ + private async getPrometheusAvgMetric({ + query, + startTimestamp, + endTimestamp + }: { + query: string; + startTimestamp: number; + endTimestamp: number; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + }): Promise { + // Construct the request body + const requestBody = new URLSearchParams({ + query, + start: startTimestamp.toString(), + end: endTimestamp.toString(), + step: `10m` // It should be higher than the time range so it returns only one value + }).toString(); + + return Math.round( + parseFloat( + ( + await this.request({ + method: "POST", + endpoint: `/api/v1/query_range`, + headers: { + "Content-Type": "application/x-www-form-urlencoded" + }, + body: requestBody + }) + ).data.result[0].values[0][1] + ) + ); + } + + private calculateEpochTimestamps(epoch: number): { startTimestamp: number; endTimestamp: number } { + const startTimestamp = this.minGenesisTime + epoch * this.slotsPerEpoch * this.secondsPerSlot; + const endTimestamp = startTimestamp + this.slotsPerEpoch * this.secondsPerSlot - 1; + return { + startTimestamp, + endTimestamp + }; + } +} diff --git a/packages/brain/src/modules/apiClients/prometheus/types.ts b/packages/brain/src/modules/apiClients/prometheus/types.ts new file mode 100644 index 00000000..bf91ce07 --- /dev/null +++ b/packages/brain/src/modules/apiClients/prometheus/types.ts @@ -0,0 +1,8 @@ +export interface AvgHostMetrics { + avgCpuTemperature: number; + avgCpuUsage: number; + avgMemoryUsage: number; + ioUtilizationPerDisk: { + [disk: string]: number; + }; +} diff --git a/packages/brain/src/modules/apiClients/standard.ts b/packages/brain/src/modules/apiClients/standard.ts index b2abbc2b..dd4c2422 100644 --- a/packages/brain/src/modules/apiClients/standard.ts +++ b/packages/brain/src/modules/apiClients/standard.ts @@ -128,9 +128,11 @@ export class StandardApi { } } else { let errorMessage = ""; + if (res.headers["content-type"] && res.headers["content-type"].includes("application/json")) { try { - errorMessage = JSON.parse(Buffer.concat(data).toString())?.message; + // if its a error message in JSON we dont know the object format so print it in string format the whole error + errorMessage = Buffer.concat(data).toString(); } catch (e) { logger.error( `Error parsing response from ${this.requestOptions.hostname} ${endpoint} ${e.message}`, diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/fetchAndInsertValidatorsPerformanceData.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/fetchAndInsertValidatorsPerformanceData.ts index 59767466..c800f206 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/fetchAndInsertValidatorsPerformanceData.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/fetchAndInsertValidatorsPerformanceData.ts @@ -17,7 +17,7 @@ import { import { BeaconchainApiError } from "../../apiClients/beaconchain/error.js"; import { BrainDbError } from "../../db/error.js"; import { ExecutionOfflineError, NodeSyncingError } from "./error.js"; -import { DappmanagerApi } from "../../apiClients/index.js"; +import { DappmanagerApi, PrometheusApi } from "../../apiClients/index.js"; import { sendValidatorsPerformanceNotifications } from "./sendValidatorsPerformanceNotifications.js"; let lastProcessedEpoch: number | undefined = undefined; @@ -29,6 +29,7 @@ export async function fetchAndInsertValidatorsPerformanceData({ executionClient, consensusClient, currentEpoch, + prometheusApi, dappmanagerApi, sendNotification }: { @@ -38,6 +39,7 @@ export async function fetchAndInsertValidatorsPerformanceData({ executionClient: ExecutionClient; consensusClient: ConsensusClient; currentEpoch: number; + prometheusApi: PrometheusApi; dappmanagerApi: DappmanagerApi; sendNotification: boolean; }): Promise { @@ -92,6 +94,7 @@ export async function fetchAndInsertValidatorsPerformanceData({ // Send notifications if the last epoch was processed without an error if (sendNotification && !validatorPerformanceError) await sendValidatorsPerformanceNotifications({ + prometheusApi, dappmanagerApi, currentEpoch: currentEpoch.toString(), validatorBlockStatusMap, diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts index 11cb7902..f4a6fb13 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts @@ -1,5 +1,5 @@ import { ExecutionClient, ConsensusClient } from "@stakingbrain/common"; -import { PostgresClient, BeaconchainApi, DappmanagerApi } from "../../apiClients/index.js"; +import { PostgresClient, BeaconchainApi, DappmanagerApi, PrometheusApi } from "../../apiClients/index.js"; import { BrainDataBase } from "../../db/index.js"; import logger from "../../logger/index.js"; import { fetchAndInsertValidatorsPerformanceData } from "./fetchAndInsertValidatorsPerformanceData.js"; @@ -11,6 +11,7 @@ export async function trackValidatorsPerformanceCron({ executionClient, consensusClient, dappmanagerApi, + prometheusApi, sendNotification }: { brainDb: BrainDataBase; @@ -19,6 +20,7 @@ export async function trackValidatorsPerformanceCron({ executionClient: ExecutionClient; consensusClient: ConsensusClient; dappmanagerApi: DappmanagerApi; + prometheusApi: PrometheusApi; sendNotification: boolean; }): Promise { try { @@ -39,6 +41,7 @@ export async function trackValidatorsPerformanceCron({ executionClient, consensusClient, currentEpoch, + prometheusApi, dappmanagerApi, sendNotification }); diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts index cf30cd2b..9a93a882 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts @@ -1,4 +1,4 @@ -import { DappmanagerApi } from "../../apiClients/index.js"; +import { DappmanagerApi, PrometheusApi } from "../../apiClients/index.js"; import { NotificationType } from "../../apiClients/dappmanager/types.js"; import { BlockProposalStatus } from "../../apiClients/postgres/types.js"; import logger from "../../logger/index.js"; @@ -12,11 +12,13 @@ import { IdealRewards, TotalRewards } from "../../apiClients/types.js"; * - danger: validator(s) missed a block */ export async function sendValidatorsPerformanceNotifications({ + prometheusApi, dappmanagerApi, currentEpoch, validatorBlockStatusMap, validatorAttestationsRewards }: { + prometheusApi: PrometheusApi; dappmanagerApi: DappmanagerApi; currentEpoch: string; validatorBlockStatusMap?: Map; @@ -27,11 +29,12 @@ export async function sendValidatorsPerformanceNotifications({ await Promise.all([ sendSuccessNotificationNotThrow({ dappmanagerApi, validatorBlockStatusMap, currentEpoch }), sendWarningNotificationNotThrow({ + prometheusApi, dappmanagerApi, validatorAttestationsRewards, currentEpoch }), - sendDangerNotificationNotThrow({ dappmanagerApi, validatorBlockStatusMap, currentEpoch }) + sendDangerNotificationNotThrow({ prometheusApi, dappmanagerApi, validatorBlockStatusMap, currentEpoch }) ]); } @@ -59,10 +62,12 @@ async function sendSuccessNotificationNotThrow({ } async function sendWarningNotificationNotThrow({ + prometheusApi, dappmanagerApi, validatorAttestationsRewards, currentEpoch }: { + prometheusApi: PrometheusApi; dappmanagerApi: DappmanagerApi; validatorAttestationsRewards: { totalRewards: TotalRewards[]; idealRewards: IdealRewards }; currentEpoch: string; @@ -72,20 +77,24 @@ async function sendWarningNotificationNotThrow({ .map((validator) => validator.validator_index); if (validatorsMissedAttestations.length === 0) return; + + const hostMetricsMessage = await getHostMetricsMessage(prometheusApi, currentEpoch); await dappmanagerApi .sendDappmanagerNotification({ title: `Missed attestation in epoch ${currentEpoch}`, notificationType: NotificationType.Warning, - body: `Validator(s) ${validatorsMissedAttestations.join(", ")} missed attestations` + body: `Validator(s) ${validatorsMissedAttestations.join(", ")} missed an attestation\n${hostMetricsMessage}` }) .catch((error) => logger.error(`${logPrefix}Failed to send warning notification to dappmanager`, error)); } async function sendDangerNotificationNotThrow({ + prometheusApi, dappmanagerApi, currentEpoch, validatorBlockStatusMap }: { + prometheusApi: PrometheusApi; dappmanagerApi: DappmanagerApi; validatorBlockStatusMap: Map; currentEpoch: string; @@ -95,11 +104,23 @@ async function sendDangerNotificationNotThrow({ ); if (validatorsMissedBlocks.length === 0) return; + const hostMetricsMessage = await getHostMetricsMessage(prometheusApi, currentEpoch); await dappmanagerApi .sendDappmanagerNotification({ title: `Block missed in epoch ${currentEpoch}`, notificationType: NotificationType.Danger, - body: `Validator(s) ${validatorsMissedBlocks.join(", ")} missed a block` + body: `Validator(s) ${validatorsMissedBlocks.join(", ")} missed a block\n${hostMetricsMessage}` }) .catch((error) => logger.error(`${logPrefix}Failed to send danger notification to dappmanager`, error)); } + +async function getHostMetricsMessage(prometheusApi: PrometheusApi, epoch: string): Promise { + const { avgCpuTemperature, avgCpuUsage, avgMemoryUsage, ioUtilizationPerDisk } = + await prometheusApi.getPrometheusMetrics({ epoch: parseInt(epoch) }); + + return `Host metrics: +- CPU temperature: ${avgCpuTemperature}°C +- CPU usage: ${avgCpuUsage}% +- Memory usage: ${avgMemoryUsage}% +- Disk I/O utilization: ${JSON.stringify(ioUtilizationPerDisk)}`; +} diff --git a/packages/brain/test/unit/modules/apiClients/prometheus.unit.test.ts b/packages/brain/test/unit/modules/apiClients/prometheus.unit.test.ts new file mode 100644 index 00000000..972d1cd4 --- /dev/null +++ b/packages/brain/test/unit/modules/apiClients/prometheus.unit.test.ts @@ -0,0 +1,17 @@ +import { Network } from "@stakingbrain/common"; +import { PrometheusApi } from "../../../../src/modules/apiClients/index.js"; + +describe.skip("Prometheus API", () => { + const prometheusApi = new PrometheusApi({ + baseUrl: "http://prometheus.dms.dappnode:9090", + minGenesisTime: 1695902100, + secondsPerSlot: 12, + slotsPerEpoch: 32, + network: Network.Holesky + }); + + it("should get prometheus metrics", async () => { + const metrics = await prometheusApi.getPrometheusMetrics({ epoch: 83660 }); + console.log(metrics); + }); +}); From 2f2132da6e33580b970bf53c6a113b4dcad7761f Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 13:07:34 +0200 Subject: [PATCH 2/7] update prometheus api url --- packages/brain/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts index c853ff05..4fc6c087 100644 --- a/packages/brain/src/index.ts +++ b/packages/brain/src/index.ts @@ -59,7 +59,7 @@ logger.debug( // Create API instances. Must preceed db initialization export const prometheusApi = new PrometheusApi({ - baseUrl: executionClientUrl, + baseUrl: "http://prometheus.dms.dappnode:9090", minGenesisTime, secondsPerSlot, slotsPerEpoch, From 073216cd022ed8fb37b8356e4a6e6cc6fe684fb1 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 13:53:44 +0200 Subject: [PATCH 3/7] append dms dashboards urls --- .../modules/apiClients/prometheus/index.ts | 2 ++ .../modules/apiClients/prometheus/types.ts | 2 ++ .../sendValidatorsPerformanceNotifications.ts | 25 +++++++++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/brain/src/modules/apiClients/prometheus/index.ts b/packages/brain/src/modules/apiClients/prometheus/index.ts index c196be75..46775b70 100644 --- a/packages/brain/src/modules/apiClients/prometheus/index.ts +++ b/packages/brain/src/modules/apiClients/prometheus/index.ts @@ -56,6 +56,8 @@ export class PrometheusApi extends StandardApi { }, {}); return { + startTimestamp, + endTimestamp, avgCpuTemperature: await this.getPrometheusAvgMetric({ query: `avg_over_time(dappmanager_cpu_temperature_celsius{app="dappmanager-custom-metrics", instance="dappmanager.dappnode:80", job="manager_sd", package="dappmanager.dnp.dappnode.eth", service="dappmanager", type="current"}[${endTimestamp - startTimestamp}s])`, startTimestamp, diff --git a/packages/brain/src/modules/apiClients/prometheus/types.ts b/packages/brain/src/modules/apiClients/prometheus/types.ts index bf91ce07..7320886c 100644 --- a/packages/brain/src/modules/apiClients/prometheus/types.ts +++ b/packages/brain/src/modules/apiClients/prometheus/types.ts @@ -1,4 +1,6 @@ export interface AvgHostMetrics { + startTimestamp: number; + endTimestamp: number; avgCpuTemperature: number; avgCpuUsage: number; avgMemoryUsage: number; diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts index 9a93a882..46670e37 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts @@ -79,6 +79,7 @@ async function sendWarningNotificationNotThrow({ if (validatorsMissedAttestations.length === 0) return; const hostMetricsMessage = await getHostMetricsMessage(prometheusApi, currentEpoch); + await dappmanagerApi .sendDappmanagerNotification({ title: `Missed attestation in epoch ${currentEpoch}`, @@ -115,12 +116,32 @@ async function sendDangerNotificationNotThrow({ } async function getHostMetricsMessage(prometheusApi: PrometheusApi, epoch: string): Promise { - const { avgCpuTemperature, avgCpuUsage, avgMemoryUsage, ioUtilizationPerDisk } = + const { startTimestamp, endTimestamp, avgCpuTemperature, avgCpuUsage, avgMemoryUsage, ioUtilizationPerDisk } = await prometheusApi.getPrometheusMetrics({ epoch: parseInt(epoch) }); + // create beautiful message for ioUtilizationPerDisk + const ioUtilizationPerDiskMessage = Object.entries(ioUtilizationPerDisk) + .map(([disk, utilization]) => { + return ` - ${disk}: ${utilization}%\n`; + }) + .join("\n"); + return `Host metrics: - CPU temperature: ${avgCpuTemperature}°C - CPU usage: ${avgCpuUsage}% - Memory usage: ${avgMemoryUsage}% -- Disk I/O utilization: ${JSON.stringify(ioUtilizationPerDisk)}`; +- Disk I/O utilization:\n${ioUtilizationPerDiskMessage} +${getDmsDashboardsMessage({ startTimestamp, endTimestamp })}`; +} + +function getDmsDashboardsMessage({ + startTimestamp, + endTimestamp +}: { + startTimestamp: number; + endTimestamp: number; +}): string { + return `For more details, check the DMS dashboards: +- [Host dashboard](http://dms.dappnode/d/dms-host/host?orgId=1&from=${startTimestamp}&to=${endTimestamp}) +- [Docker dashboard](http://dms.dappnode/d/dms-docker/docker?orgId=1&from=${startTimestamp}&to=${endTimestamp})`; } From 1c0fd859b8c5d5554d2e5448ce1d88632fdfe969 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 13:56:05 +0200 Subject: [PATCH 4/7] formate message --- .../sendValidatorsPerformanceNotifications.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts index 46670e37..c33c8a69 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts @@ -126,11 +126,11 @@ async function getHostMetricsMessage(prometheusApi: PrometheusApi, epoch: string }) .join("\n"); - return `Host metrics: + return `Average host metrics within epoch ${epoch}:\n - CPU temperature: ${avgCpuTemperature}°C - CPU usage: ${avgCpuUsage}% - Memory usage: ${avgMemoryUsage}% -- Disk I/O utilization:\n${ioUtilizationPerDiskMessage} +- Disk I/O utilization:\n${ioUtilizationPerDiskMessage}\n ${getDmsDashboardsMessage({ startTimestamp, endTimestamp })}`; } @@ -141,7 +141,7 @@ function getDmsDashboardsMessage({ startTimestamp: number; endTimestamp: number; }): string { - return `For more details, check the DMS dashboards: + return `For more details, check the DMS dashboards:\n - [Host dashboard](http://dms.dappnode/d/dms-host/host?orgId=1&from=${startTimestamp}&to=${endTimestamp}) - [Docker dashboard](http://dms.dappnode/d/dms-docker/docker?orgId=1&from=${startTimestamp}&to=${endTimestamp})`; } From 8f7382cb71c0d0d1ed8e691dcacd802229327231 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 14:01:59 +0200 Subject: [PATCH 5/7] fix formatting --- .../sendValidatorsPerformanceNotifications.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts index c33c8a69..51309557 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts @@ -122,7 +122,7 @@ async function getHostMetricsMessage(prometheusApi: PrometheusApi, epoch: string // create beautiful message for ioUtilizationPerDisk const ioUtilizationPerDiskMessage = Object.entries(ioUtilizationPerDisk) .map(([disk, utilization]) => { - return ` - ${disk}: ${utilization}%\n`; + return ` - ${disk}: ${utilization}%`; }) .join("\n"); @@ -130,7 +130,7 @@ async function getHostMetricsMessage(prometheusApi: PrometheusApi, epoch: string - CPU temperature: ${avgCpuTemperature}°C - CPU usage: ${avgCpuUsage}% - Memory usage: ${avgMemoryUsage}% -- Disk I/O utilization:\n${ioUtilizationPerDiskMessage}\n +- Disk I/O utilization:\n${ioUtilizationPerDiskMessage}\n\n ${getDmsDashboardsMessage({ startTimestamp, endTimestamp })}`; } From 96e96896b6b52d88b535b20e7cb718abd939b4a5 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 14:05:26 +0200 Subject: [PATCH 6/7] use timestamp in ms --- .../sendValidatorsPerformanceNotifications.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts index 51309557..74b72372 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts @@ -141,7 +141,11 @@ function getDmsDashboardsMessage({ startTimestamp: number; endTimestamp: number; }): string { + // dashboard links must be with timestamps in milliseconds + // see https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/manage-dashboard-links/ + const startTimestampInMs = startTimestamp * 1000; + const endTimestampInMs = endTimestamp * 1000; return `For more details, check the DMS dashboards:\n -- [Host dashboard](http://dms.dappnode/d/dms-host/host?orgId=1&from=${startTimestamp}&to=${endTimestamp}) -- [Docker dashboard](http://dms.dappnode/d/dms-docker/docker?orgId=1&from=${startTimestamp}&to=${endTimestamp})`; +- [Host dashboard](http://dms.dappnode/d/dms-host/host?orgId=1&from=${startTimestampInMs}&to=${endTimestampInMs}) +- [Docker dashboard](http://dms.dappnode/d/dms-docker/docker?orgId=1&from=${startTimestampInMs}&to=${endTimestampInMs})`; } From 31c2287cd7f4659da3c1daf2215181e6f72d9b55 Mon Sep 17 00:00:00 2001 From: pablomendezroyo Date: Fri, 4 Oct 2024 14:07:47 +0200 Subject: [PATCH 7/7] make message more beautiful --- .../sendValidatorsPerformanceNotifications.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts index 74b72372..8b4b0cd1 100644 --- a/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts +++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/sendValidatorsPerformanceNotifications.ts @@ -119,18 +119,17 @@ async function getHostMetricsMessage(prometheusApi: PrometheusApi, epoch: string const { startTimestamp, endTimestamp, avgCpuTemperature, avgCpuUsage, avgMemoryUsage, ioUtilizationPerDisk } = await prometheusApi.getPrometheusMetrics({ epoch: parseInt(epoch) }); - // create beautiful message for ioUtilizationPerDisk + // Create a formatted message for Disk I/O utilization const ioUtilizationPerDiskMessage = Object.entries(ioUtilizationPerDisk) - .map(([disk, utilization]) => { - return ` - ${disk}: ${utilization}%`; - }) + .map(([disk, utilization]) => ` - *${disk}*: *${utilization}%*`) .join("\n"); - return `Average host metrics within epoch ${epoch}:\n -- CPU temperature: ${avgCpuTemperature}°C -- CPU usage: ${avgCpuUsage}% -- Memory usage: ${avgMemoryUsage}% -- Disk I/O utilization:\n${ioUtilizationPerDiskMessage}\n\n + // Create a structured and formatted message + return `⚠️ *Average host metrics within epoch ${epoch}*:\n +- *CPU temperature*: *${avgCpuTemperature}°C* +- *CPU usage*: *${avgCpuUsage}%* +- *Memory usage*: *${avgMemoryUsage}%* +- *Disk I/O utilization*:\n${ioUtilizationPerDiskMessage}\n ${getDmsDashboardsMessage({ startTimestamp, endTimestamp })}`; } @@ -141,11 +140,10 @@ function getDmsDashboardsMessage({ startTimestamp: number; endTimestamp: number; }): string { - // dashboard links must be with timestamps in milliseconds - // see https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/manage-dashboard-links/ const startTimestampInMs = startTimestamp * 1000; const endTimestampInMs = endTimestamp * 1000; - return `For more details, check the DMS dashboards:\n + + return `🔗 *For more details, check the DMS dashboards:*\n - [Host dashboard](http://dms.dappnode/d/dms-host/host?orgId=1&from=${startTimestampInMs}&to=${endTimestampInMs}) - [Docker dashboard](http://dms.dappnode/d/dms-docker/docker?orgId=1&from=${startTimestampInMs}&to=${endTimestampInMs})`; }