From 5a8bc3bf7da316e72dce04ca318f31e1fda443a6 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Tue, 31 May 2022 16:13:44 +0900 Subject: [PATCH 01/31] Add otel metrics to alerting plugin --- package.json | 3 + .../server/monitoring/in_memory_metrics.ts | 25 +++- .../server/task_runner/task_runner.ts | 2 + yarn.lock | 139 +++++++++++++++++- 4 files changed, 165 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c6d3d11790b17..6547dd6a7327c 100644 --- a/package.json +++ b/package.json @@ -208,6 +208,9 @@ "@mapbox/mapbox-gl-draw": "1.3.0", "@mapbox/mapbox-gl-rtl-text": "0.2.3", "@mapbox/vector-tile": "1.3.1", + "@opentelemetry/api-metrics": "0.27.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.27.0", + "@opentelemetry/sdk-metrics-base": "^0.27.0", "@reduxjs/toolkit": "^1.6.1", "@slack/webhook": "^5.0.4", "@turf/along": "6.0.1", diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index a2d0425da1427..e5a941e69f2b9 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -6,6 +6,9 @@ */ import { Logger } from '@kbn/logging'; +import { MeterProvider } from '@opentelemetry/sdk-metrics-base'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; +import { Attributes, Counter } from '@opentelemetry/api-metrics'; export enum IN_MEMORY_METRICS { RULE_EXECUTIONS = 'ruleExecutions', @@ -21,11 +24,29 @@ export class InMemoryMetrics { [IN_MEMORY_METRICS.RULE_TIMEOUTS]: 0, }; + private otelMetrics: { + [IN_MEMORY_METRICS.RULE_EXECUTIONS]: Counter; + [IN_MEMORY_METRICS.RULE_FAILURES]: Counter; + [IN_MEMORY_METRICS.RULE_TIMEOUTS]: Counter; + }; + constructor(logger: Logger) { this.logger = logger; + + this.logger.debug('MATSCHAFFER: CREATING meter provider'); + const provider = new MeterProvider({ + exporter: new OTLPMetricExporter(), + interval: 1000, + }); + const meter = provider.getMeter('example-meter'); + this.otelMetrics = { + [IN_MEMORY_METRICS.RULE_EXECUTIONS]: meter.createCounter(IN_MEMORY_METRICS.RULE_EXECUTIONS), + [IN_MEMORY_METRICS.RULE_FAILURES]: meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), + [IN_MEMORY_METRICS.RULE_TIMEOUTS]: meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), + }; } - public increment(metric: IN_MEMORY_METRICS) { + public increment(metric: IN_MEMORY_METRICS, attributes?: Attributes) { if (this.inMemoryMetrics[metric] === null) { this.logger.info( `Metric ${metric} is null because the counter ran over the max safe integer value, skipping increment.` @@ -41,6 +62,8 @@ export class InMemoryMetrics { } else { (this.inMemoryMetrics[metric] as number)++; } + + this.otelMetrics[metric].add(1, attributes); } public getInMemoryMetric(metric: IN_MEMORY_METRICS) { diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index bd83b269ce10d..8c8064ab75587 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -797,6 +797,8 @@ export class TaskRunner< }; if (!this.cancelled) { + this.logger.debug("MATSCHAFFER: INCREMENTING rule's execution count"); + // this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS, { rule: this.ruleType.id }); this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); if (executionStatus.error) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); diff --git a/yarn.lock b/yarn.lock index dd9f1769f42bb..29b5f2901038f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2001,6 +2001,25 @@ resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== +"@grpc/grpc-js@^1.3.7": + version "1.6.7" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.6.7.tgz#4c4fa998ff719fe859ac19fe977fdef097bb99aa" + integrity sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw== + dependencies: + "@grpc/proto-loader" "^0.6.4" + "@types/node" ">=12.12.47" + +"@grpc/proto-loader@^0.6.4": + version "0.6.12" + resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.6.12.tgz#459b619b8b9b67794bf0d1cb819653a38c63e164" + integrity sha512-filTVbETFnxb9CyRX98zN18ilChTuf/C5scZ2xyaOTp0EHGq0/ufX8rjqXUcSb1Gpv7eZq4M2jDvbh9BogKnrg== + dependencies: + "@types/long" "^4.0.1" + lodash.camelcase "^4.3.0" + long "^4.0.0" + protobufjs "^6.10.0" + yargs "^16.2.0" + "@gulp-sourcemaps/identity-map@1.X": version "1.0.2" resolved "https://registry.yarnpkg.com/@gulp-sourcemaps/identity-map/-/identity-map-1.0.2.tgz#1e6fe5d8027b1f285dc0d31762f566bccd73d5a9" @@ -4026,11 +4045,101 @@ dependencies: "@octokit/openapi-types" "^11.2.0" +"@opentelemetry/api-metrics@0.27.0": + version "0.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-metrics/-/api-metrics-0.27.0.tgz#d8eca344ed1155f3ea8a8133ade827b4bb90efbf" + integrity sha512-tB79288bwjkdhPNpw4UdOEy3bacVwtol6Que7cAu8KEJ9ULjRfSiwpYEwJY/oER3xZ7zNFz0uiJ7N1jSiotpVA== + "@opentelemetry/api@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.1.0.tgz#563539048255bbe1a5f4f586a4a10a1bb737f44a" integrity sha512-hf+3bwuBwtXsugA2ULBc95qxrOqP2pOekLz34BJhcAKawt94vfeNyUKpYc0lZQ/3sCP6LqRa7UAdHA7i5UODzQ== +"@opentelemetry/core@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.0.1.tgz#5e08cef21946fdea7952f544e8f667f6d2a0ded8" + integrity sha512-90nQ2X6b/8X+xjcLDBYKooAcOsIlwLRYm+1VsxcX5cHl6V4CSVmDpBreQSDH/A21SqROzapk6813008SatmPpQ== + dependencies: + "@opentelemetry/semantic-conventions" "1.0.1" + +"@opentelemetry/exporter-metrics-otlp-grpc@^0.27.0": + version "0.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.27.0.tgz#a4f3f76a719e84bb1a6cd5d857a8eb94e20195a1" + integrity sha512-vAAsFZNzeBldxpx/D5zfSypcPrrd0/MNmrrdduCzh9ysgOK5TC52lh2pQBjIRv/q0GPRwmMBsdSLpjJuLDQcCg== + dependencies: + "@grpc/grpc-js" "^1.3.7" + "@grpc/proto-loader" "^0.6.4" + "@opentelemetry/core" "1.0.1" + "@opentelemetry/exporter-metrics-otlp-http" "0.27.0" + "@opentelemetry/exporter-trace-otlp-grpc" "0.27.0" + "@opentelemetry/exporter-trace-otlp-http" "0.27.0" + "@opentelemetry/resources" "1.0.1" + "@opentelemetry/sdk-metrics-base" "0.27.0" + +"@opentelemetry/exporter-metrics-otlp-http@0.27.0": + version "0.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.27.0.tgz#1ae4d981e25ab15cc2a78a44971ea82cb6315eb9" + integrity sha512-97eAPtA9po403CpNnd4n8HtEN6HbXwb05eUL5Jxaj70RSKAHBKjAFG4/Bk75DX3G+glHN5Gm/waMmcuBqFW5rA== + dependencies: + "@opentelemetry/api-metrics" "0.27.0" + "@opentelemetry/core" "1.0.1" + "@opentelemetry/exporter-trace-otlp-http" "0.27.0" + "@opentelemetry/resources" "1.0.1" + "@opentelemetry/sdk-metrics-base" "0.27.0" + +"@opentelemetry/exporter-trace-otlp-grpc@0.27.0": + version "0.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.27.0.tgz#1fdb7b642ab17bff8dbc96eeb5c839dd55354a98" + integrity sha512-fFoLoCv9beWRouuhLy8zqnHrPj+Bj89iUbUzcg80cQ4tP3AXKyjWBowk/xHteKsTFbQYkIBhIQOpekyHtExwRw== + dependencies: + "@grpc/grpc-js" "^1.3.7" + "@grpc/proto-loader" "^0.6.4" + "@opentelemetry/core" "1.0.1" + "@opentelemetry/exporter-trace-otlp-http" "0.27.0" + "@opentelemetry/resources" "1.0.1" + "@opentelemetry/sdk-trace-base" "1.0.1" + +"@opentelemetry/exporter-trace-otlp-http@0.27.0": + version "0.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.27.0.tgz#0d2798e89ecae1487c3a2e7fed45eec7157df7ee" + integrity sha512-ZE8Ns/GGW83E4igrby69shiqEkVo+cULzbm4DprSEMCWrPAL/NBobETFOiOQyBBBgIfrhi5EG6truUiafB1cMQ== + dependencies: + "@opentelemetry/core" "1.0.1" + "@opentelemetry/resources" "1.0.1" + "@opentelemetry/sdk-trace-base" "1.0.1" + +"@opentelemetry/resources@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.0.1.tgz#2d190e2e6e64327b436447a8dd799afc673b6e07" + integrity sha512-p8DevOaAEepPucUtImR4cZKHOE2L1jgQAtkdZporV+XnxPA/HqCHPEESyUVuo4f5M0NUlL6k5Pba75KwNJlTRg== + dependencies: + "@opentelemetry/core" "1.0.1" + "@opentelemetry/semantic-conventions" "1.0.1" + +"@opentelemetry/sdk-metrics-base@0.27.0", "@opentelemetry/sdk-metrics-base@^0.27.0": + version "0.27.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics-base/-/sdk-metrics-base-0.27.0.tgz#cbd9189dd427d445c39aa3981fcc769891db1918" + integrity sha512-HpiWI4sVNsjp3FGyUlc24KvUY2Whl4PQVwcbA/gWv2kHaLQrDJrWC+3rjUR+87Mrd0nsiqJ85xhGFU6IK8h7gg== + dependencies: + "@opentelemetry/api-metrics" "0.27.0" + "@opentelemetry/core" "1.0.1" + "@opentelemetry/resources" "1.0.1" + lodash.merge "^4.6.2" + +"@opentelemetry/sdk-trace-base@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.0.1.tgz#b88c72ac768eed58baab41552ce9070c57d5b7bf" + integrity sha512-JVSAepTpW7dnqfV7XFN0zHj1jXGNd5OcvIGQl76buogqffdgJdgJWQNrOuUJaus56zrOtlzqFH+YtMA9RGEg8w== + dependencies: + "@opentelemetry/core" "1.0.1" + "@opentelemetry/resources" "1.0.1" + "@opentelemetry/semantic-conventions" "1.0.1" + +"@opentelemetry/semantic-conventions@1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.0.1.tgz#9349c3860a53468fa2108b5df09aa843f22dbf94" + integrity sha512-7XU1sfQ8uCVcXLxtAHA8r3qaLJ2oq7sKtEwzZhzuEXqYmjW+n+J4yM3kNo0HQo3Xp1eUe47UM6Wy6yuAvIyllg== + "@percy/agent@^0.28.6": version "0.28.6" resolved "https://registry.yarnpkg.com/@percy/agent/-/agent-0.28.6.tgz#b220fab6ddcf63ae4e6c343108ba6955a772ce1c" @@ -6566,6 +6675,11 @@ resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== +"@types/long@^4.0.1": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + "@types/lru-cache@^5.1.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" @@ -6733,7 +6847,7 @@ dependencies: "@types/node" "*" -"@types/node@*", "@types/node@12.20.24", "@types/node@16.11.7", "@types/node@>= 8", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^14.0.10", "@types/node@^14.14.31": +"@types/node@*", "@types/node@12.20.24", "@types/node@16.11.7", "@types/node@>= 8", "@types/node@>=12.12.47", "@types/node@>=13.7.0", "@types/node@>=8.9.0", "@types/node@^10.1.0", "@types/node@^14.0.10", "@types/node@^14.14.31": version "16.11.7" resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.7.tgz#36820945061326978c42a01e56b61cd223dfdc42" integrity sha512-QB5D2sqfSjCmTuWcBWyJ+/44bcjO7VbjSbOE0ucoVbAsSNQc4Lt6QkgkVXkTDwkL4z/beecZNDvVX15D4P8Jbw== @@ -23157,6 +23271,25 @@ protobufjs@6.8.8: "@types/node" "^10.1.0" long "^4.0.0" +protobufjs@^6.10.0: + version "6.11.3" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.3.tgz#637a527205a35caa4f3e2a9a4a13ddffe0e7af74" + integrity sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.1" + "@types/node" ">=13.7.0" + long "^4.0.0" + protocol-buffers-schema@^3.3.1: version "3.3.2" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.3.2.tgz#00434f608b4e8df54c59e070efeefc37fb4bb859" @@ -30299,9 +30432,9 @@ yargs-unparser@2.0.0: flat "^5.0.2" is-plain-obj "^2.1.0" -yargs@16.2.0: +yargs@16.2.0, yargs@^16.2.0: version "16.2.0" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" + resolved "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz" integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== dependencies: cliui "^7.0.2" From 08b66fb8bf94cd374c6ec2c1efa60991621d3383 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Tue, 31 May 2022 22:02:05 +0900 Subject: [PATCH 02/31] Hacking around test failures --- x-pack/plugins/alerting/jest.config.js | 2 +- .../alerting/server/monitoring/in_memory_metrics.test.ts | 2 ++ x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts | 2 +- x-pack/plugins/alerting/server/task_runner/task_runner.ts | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/alerting/jest.config.js b/x-pack/plugins/alerting/jest.config.js index 05db974299b40..257dbcd2cddd9 100644 --- a/x-pack/plugins/alerting/jest.config.js +++ b/x-pack/plugins/alerting/jest.config.js @@ -6,7 +6,7 @@ */ module.exports = { - preset: '@kbn/test', + preset: '@kbn/test/jest_node', rootDir: '../../..', roots: ['/x-pack/plugins/alerting'], coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/alerting', diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts index 6a139df0aa844..8cb410fb26011 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts @@ -8,6 +8,8 @@ import { InMemoryMetrics, IN_MEMORY_METRICS } from '.'; import { loggingSystemMock } from '@kbn/core/server/mocks'; +jest.useFakeTimers(); + describe('inMemoryMetrics', () => { const logger = loggingSystemMock.createLogger(); const inMemoryMetrics = new InMemoryMetrics(logger); diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index e5a941e69f2b9..5818230c877d5 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -33,7 +33,7 @@ export class InMemoryMetrics { constructor(logger: Logger) { this.logger = logger; - this.logger.debug('MATSCHAFFER: CREATING meter provider'); + // this.logger.debug('MATSCHAFFER: CREATING meter provider'); const provider = new MeterProvider({ exporter: new OTLPMetricExporter(), interval: 1000, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 8c8064ab75587..94d057f543453 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -797,7 +797,7 @@ export class TaskRunner< }; if (!this.cancelled) { - this.logger.debug("MATSCHAFFER: INCREMENTING rule's execution count"); + // this.logger.debug("MATSCHAFFER: INCREMENTING rule's execution count"); // this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS, { rule: this.ruleType.id }); this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); if (executionStatus.error) { From 49c6f7f24039532564ae66e52dec14e06be0bb82 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 2 Jun 2022 11:32:26 +0900 Subject: [PATCH 03/31] Revert "Hacking around test failures" This reverts commit 08b66fb8bf94cd374c6ec2c1efa60991621d3383. --- x-pack/plugins/alerting/jest.config.js | 2 +- .../alerting/server/monitoring/in_memory_metrics.test.ts | 2 -- x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts | 2 +- x-pack/plugins/alerting/server/task_runner/task_runner.ts | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/alerting/jest.config.js b/x-pack/plugins/alerting/jest.config.js index 257dbcd2cddd9..05db974299b40 100644 --- a/x-pack/plugins/alerting/jest.config.js +++ b/x-pack/plugins/alerting/jest.config.js @@ -6,7 +6,7 @@ */ module.exports = { - preset: '@kbn/test/jest_node', + preset: '@kbn/test', rootDir: '../../..', roots: ['/x-pack/plugins/alerting'], coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/alerting', diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts index 8cb410fb26011..6a139df0aa844 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.test.ts @@ -8,8 +8,6 @@ import { InMemoryMetrics, IN_MEMORY_METRICS } from '.'; import { loggingSystemMock } from '@kbn/core/server/mocks'; -jest.useFakeTimers(); - describe('inMemoryMetrics', () => { const logger = loggingSystemMock.createLogger(); const inMemoryMetrics = new InMemoryMetrics(logger); diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index 5818230c877d5..e5a941e69f2b9 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -33,7 +33,7 @@ export class InMemoryMetrics { constructor(logger: Logger) { this.logger = logger; - // this.logger.debug('MATSCHAFFER: CREATING meter provider'); + this.logger.debug('MATSCHAFFER: CREATING meter provider'); const provider = new MeterProvider({ exporter: new OTLPMetricExporter(), interval: 1000, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 94d057f543453..8c8064ab75587 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -797,7 +797,7 @@ export class TaskRunner< }; if (!this.cancelled) { - // this.logger.debug("MATSCHAFFER: INCREMENTING rule's execution count"); + this.logger.debug("MATSCHAFFER: INCREMENTING rule's execution count"); // this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS, { rule: this.ruleType.id }); this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); if (executionStatus.error) { From 92e94df3455f0e992be0091a14a6ad086d3647a4 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 2 Jun 2022 11:53:02 +0900 Subject: [PATCH 04/31] Guard otel initialization & remove debug logging Should avoid test failures. --- .../server/monitoring/in_memory_metrics.ts | 30 +++++++++++-------- .../server/task_runner/task_runner.ts | 2 -- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index e5a941e69f2b9..96315010f5554 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -24,7 +24,7 @@ export class InMemoryMetrics { [IN_MEMORY_METRICS.RULE_TIMEOUTS]: 0, }; - private otelMetrics: { + private otelMetrics?: { [IN_MEMORY_METRICS.RULE_EXECUTIONS]: Counter; [IN_MEMORY_METRICS.RULE_FAILURES]: Counter; [IN_MEMORY_METRICS.RULE_TIMEOUTS]: Counter; @@ -33,17 +33,19 @@ export class InMemoryMetrics { constructor(logger: Logger) { this.logger = logger; - this.logger.debug('MATSCHAFFER: CREATING meter provider'); - const provider = new MeterProvider({ - exporter: new OTLPMetricExporter(), - interval: 1000, - }); - const meter = provider.getMeter('example-meter'); - this.otelMetrics = { - [IN_MEMORY_METRICS.RULE_EXECUTIONS]: meter.createCounter(IN_MEMORY_METRICS.RULE_EXECUTIONS), - [IN_MEMORY_METRICS.RULE_FAILURES]: meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), - [IN_MEMORY_METRICS.RULE_TIMEOUTS]: meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), - }; + if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) { + const provider = new MeterProvider({ + exporter: new OTLPMetricExporter(), + interval: 1000, + }); + + const meter = provider.getMeter('alerting-meter'); + this.otelMetrics = { + [IN_MEMORY_METRICS.RULE_EXECUTIONS]: meter.createCounter(IN_MEMORY_METRICS.RULE_EXECUTIONS), + [IN_MEMORY_METRICS.RULE_FAILURES]: meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), + [IN_MEMORY_METRICS.RULE_TIMEOUTS]: meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), + }; + } } public increment(metric: IN_MEMORY_METRICS, attributes?: Attributes) { @@ -63,7 +65,9 @@ export class InMemoryMetrics { (this.inMemoryMetrics[metric] as number)++; } - this.otelMetrics[metric].add(1, attributes); + if (this.otelMetrics) { + this.otelMetrics[metric].add(1, attributes); + } } public getInMemoryMetric(metric: IN_MEMORY_METRICS) { diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 8c8064ab75587..bd83b269ce10d 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -797,8 +797,6 @@ export class TaskRunner< }; if (!this.cancelled) { - this.logger.debug("MATSCHAFFER: INCREMENTING rule's execution count"); - // this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS, { rule: this.ruleType.id }); this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); if (executionStatus.error) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); From 2a41cf42a158b25ef42ad1624fb1c0c2d439f8dc Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 2 Jun 2022 11:59:29 +0900 Subject: [PATCH 05/31] Follow readonly lint recommendation --- x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index 96315010f5554..36f18f110adc8 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -24,7 +24,7 @@ export class InMemoryMetrics { [IN_MEMORY_METRICS.RULE_TIMEOUTS]: 0, }; - private otelMetrics?: { + private readonly otelMetrics?: { [IN_MEMORY_METRICS.RULE_EXECUTIONS]: Counter; [IN_MEMORY_METRICS.RULE_FAILURES]: Counter; [IN_MEMORY_METRICS.RULE_TIMEOUTS]: Counter; From 634f538e9bf521fdf494d4d212b1a84f765f433c Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 2 Jun 2022 14:10:35 +0900 Subject: [PATCH 06/31] Move to 10s interval --- x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index 36f18f110adc8..0b8cf89762002 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -36,7 +36,7 @@ export class InMemoryMetrics { if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) { const provider = new MeterProvider({ exporter: new OTLPMetricExporter(), - interval: 1000, + interval: 10000, }); const meter = provider.getMeter('alerting-meter'); From 028664a86d5a6cbc717ca8851f7faf13fbcc9eef Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 6 Jun 2022 16:24:59 +0900 Subject: [PATCH 07/31] Use otel api for meter type --- .../monitoring_collection/jest.config.js | 2 +- .../monitoring_collection/server/config.ts | 7 ++++++ .../server/plugin.test.ts | 5 +++++ .../monitoring_collection/server/plugin.ts | 22 ++++++++++++++++++- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/jest.config.js b/x-pack/plugins/monitoring_collection/jest.config.js index d2bb0617857d2..1421a22541184 100644 --- a/x-pack/plugins/monitoring_collection/jest.config.js +++ b/x-pack/plugins/monitoring_collection/jest.config.js @@ -6,7 +6,7 @@ */ module.exports = { - preset: '@kbn/test', + preset: '@kbn/test/jest_node', rootDir: '../../..', roots: ['/x-pack/plugins/monitoring_collection'], coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/monitoring_collection', diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index 275d2f31e505d..0d430a562b5b7 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -9,6 +9,13 @@ import { schema, TypeOf } from '@kbn/config-schema'; export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), + opentelemetry: schema.object({ + metrics: schema.object({ + oltp: schema.object({ + url: schema.maybe(schema.string()), + }), + }), + }), }); export type MonitoringCollectionConfig = ReturnType; diff --git a/x-pack/plugins/monitoring_collection/server/plugin.test.ts b/x-pack/plugins/monitoring_collection/server/plugin.test.ts index 15b9934525b9e..d3e231cb7db83 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.test.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.test.ts @@ -124,5 +124,10 @@ describe('monitoring_collection plugin', () => { ); expect(metrics).toBeUndefined(); }); + + it('should provide an opentelemetry meter', async () => { + const { getMeter } = plugin.setup(coreSetup); + expect(getMeter('monitoring-collection-plugin-test')).toBeDefined(); + }); }); }); diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index e1c3a5064a579..45c2d5ab3706f 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -9,11 +9,16 @@ import { JsonObject } from '@kbn/utility-types'; import { CoreSetup, Plugin, PluginInitializerContext, Logger } from '@kbn/core/server'; import { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server'; import { ServiceStatus } from '@kbn/core/server'; +import { Meter } from '@opentelemetry/api-metrics'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; +import { MeterProvider } from '@opentelemetry/sdk-metrics-base'; +import { MonitoringCollectionConfig } from './config'; import { registerDynamicRoute } from './routes'; import { TYPE_ALLOWLIST } from './constants'; export interface MonitoringCollectionSetup { registerMetric: (metric: Metric) => void; + getMeter: (name: string) => Meter; } export type MetricResult = T & JsonObject; @@ -27,12 +32,15 @@ export interface Metric { export class MonitoringCollectionPlugin implements Plugin { private readonly initializerContext: PluginInitializerContext; private readonly logger: Logger; + private readonly config: MonitoringCollectionConfig; + private meterProvider?: MeterProvider; private metrics: Record> = {}; - constructor(initializerContext: PluginInitializerContext) { + constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; this.logger = initializerContext.logger.get(); + this.config = initializerContext.config.get(); } async getMetric(type: string) { @@ -66,6 +74,17 @@ export class MonitoringCollectionPlugin implements Plugin(metric: Metric) => { if (this.metrics.hasOwnProperty(metric.type)) { @@ -82,6 +101,7 @@ export class MonitoringCollectionPlugin implements Plugin Date: Mon, 6 Jun 2022 17:43:52 +0900 Subject: [PATCH 08/31] Fix mock --- x-pack/plugins/monitoring_collection/server/mocks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring_collection/server/mocks.ts b/x-pack/plugins/monitoring_collection/server/mocks.ts index 9858653df648f..4003d09624b5a 100644 --- a/x-pack/plugins/monitoring_collection/server/mocks.ts +++ b/x-pack/plugins/monitoring_collection/server/mocks.ts @@ -10,7 +10,7 @@ import { MonitoringCollectionSetup } from '.'; const createSetupMock = (): jest.Mocked => { const mock = { registerMetric: jest.fn(), - getMetrics: jest.fn(), + getMeter: jest.fn(), }; return mock; }; From 3cc35fa12a236654bbf067508ddd515a35684c30 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 6 Jun 2022 17:45:18 +0900 Subject: [PATCH 09/31] Set service name --- package.json | 1 + .../monitoring_collection/server/plugin.ts | 4 ++++ yarn.lock | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/package.json b/package.json index 0544b10729d07..37d83fa24566a 100644 --- a/package.json +++ b/package.json @@ -220,6 +220,7 @@ "@mapbox/vector-tile": "1.3.1", "@opentelemetry/api-metrics": "0.27.0", "@opentelemetry/exporter-metrics-otlp-grpc": "^0.27.0", + "@opentelemetry/resources": "^1.0.1", "@opentelemetry/sdk-metrics-base": "^0.27.0", "@reduxjs/toolkit": "^1.6.1", "@slack/webhook": "^5.0.4", diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 45c2d5ab3706f..9231495867d2e 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -12,6 +12,7 @@ import { ServiceStatus } from '@kbn/core/server'; import { Meter } from '@opentelemetry/api-metrics'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; import { MeterProvider } from '@opentelemetry/sdk-metrics-base'; +import { Resource } from '@opentelemetry/resources'; import { MonitoringCollectionConfig } from './config'; import { registerDynamicRoute } from './routes'; import { TYPE_ALLOWLIST } from './constants'; @@ -79,6 +80,9 @@ export class MonitoringCollectionPlugin implements Plugin Date: Mon, 6 Jun 2022 17:46:00 +0900 Subject: [PATCH 10/31] Use otel metrics from monitoring_collection plugin --- .../server/monitoring/in_memory_metrics.ts | 28 +++++++------------ .../monitoring/register_node_collector.ts | 3 ++ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index 0b8cf89762002..31e51995dad7f 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -6,9 +6,7 @@ */ import { Logger } from '@kbn/logging'; -import { MeterProvider } from '@opentelemetry/sdk-metrics-base'; -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; -import { Attributes, Counter } from '@opentelemetry/api-metrics'; +import { Attributes, Counter, Meter } from '@opentelemetry/api-metrics'; export enum IN_MEMORY_METRICS { RULE_EXECUTIONS = 'ruleExecutions', @@ -24,7 +22,7 @@ export class InMemoryMetrics { [IN_MEMORY_METRICS.RULE_TIMEOUTS]: 0, }; - private readonly otelMetrics?: { + private otelMetrics?: { [IN_MEMORY_METRICS.RULE_EXECUTIONS]: Counter; [IN_MEMORY_METRICS.RULE_FAILURES]: Counter; [IN_MEMORY_METRICS.RULE_TIMEOUTS]: Counter; @@ -32,20 +30,6 @@ export class InMemoryMetrics { constructor(logger: Logger) { this.logger = logger; - - if (process.env.OTEL_EXPORTER_OTLP_ENDPOINT) { - const provider = new MeterProvider({ - exporter: new OTLPMetricExporter(), - interval: 10000, - }); - - const meter = provider.getMeter('alerting-meter'); - this.otelMetrics = { - [IN_MEMORY_METRICS.RULE_EXECUTIONS]: meter.createCounter(IN_MEMORY_METRICS.RULE_EXECUTIONS), - [IN_MEMORY_METRICS.RULE_FAILURES]: meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), - [IN_MEMORY_METRICS.RULE_TIMEOUTS]: meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), - }; - } } public increment(metric: IN_MEMORY_METRICS, attributes?: Attributes) { @@ -77,4 +61,12 @@ export class InMemoryMetrics { public getAllInMemoryMetrics() { return this.inMemoryMetrics; } + + public registerMeter(meter: Meter) { + this.otelMetrics = { + [IN_MEMORY_METRICS.RULE_EXECUTIONS]: meter.createCounter(IN_MEMORY_METRICS.RULE_EXECUTIONS), + [IN_MEMORY_METRICS.RULE_FAILURES]: meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), + [IN_MEMORY_METRICS.RULE_TIMEOUTS]: meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), + }; + } } diff --git a/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts b/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts index 839787c6e78fc..20e8e83836576 100644 --- a/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts +++ b/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts @@ -15,6 +15,9 @@ export function registerNodeCollector({ monitoringCollection: MonitoringCollectionSetup; inMemoryMetrics: InMemoryMetrics; }) { + const meter = monitoringCollection.getMeter('alerting-meter'); + inMemoryMetrics.registerMeter(meter); + monitoringCollection.registerMetric({ type: 'node_rules', schema: { From b47e16a72540a7ec00a9eb780a6cddc9be3d0766 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 6 Jun 2022 23:24:51 +0900 Subject: [PATCH 11/31] Typo on otlp --- x-pack/plugins/monitoring_collection/server/config.ts | 2 +- x-pack/plugins/monitoring_collection/server/plugin.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index 0d430a562b5b7..3a9952cba12a7 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -11,7 +11,7 @@ export const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), opentelemetry: schema.object({ metrics: schema.object({ - oltp: schema.object({ + otlp: schema.object({ url: schema.maybe(schema.string()), }), }), diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 9231495867d2e..129324cef89c5 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -75,11 +75,11 @@ export class MonitoringCollectionPlugin implements Plugin Date: Mon, 6 Jun 2022 23:26:25 +0900 Subject: [PATCH 12/31] Put getMetrics back in mock --- x-pack/plugins/monitoring_collection/server/mocks.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/monitoring_collection/server/mocks.ts b/x-pack/plugins/monitoring_collection/server/mocks.ts index 4003d09624b5a..a0cb9cb1d6fe7 100644 --- a/x-pack/plugins/monitoring_collection/server/mocks.ts +++ b/x-pack/plugins/monitoring_collection/server/mocks.ts @@ -10,6 +10,7 @@ import { MonitoringCollectionSetup } from '.'; const createSetupMock = (): jest.Mocked => { const mock = { registerMetric: jest.fn(), + getMetrics: jest.fn(), getMeter: jest.fn(), }; return mock; From a71f4afe9354909fd78762a66f57342a91abce72 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Tue, 7 Jun 2022 00:03:56 +0900 Subject: [PATCH 13/31] Take out resource due to type check issue Will put it back once using latest otel metrics --- x-pack/plugins/monitoring_collection/server/plugin.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 129324cef89c5..33b36293fa728 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -12,7 +12,7 @@ import { ServiceStatus } from '@kbn/core/server'; import { Meter } from '@opentelemetry/api-metrics'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; import { MeterProvider } from '@opentelemetry/sdk-metrics-base'; -import { Resource } from '@opentelemetry/resources'; +// import { Resource } from '@opentelemetry/resources'; import { MonitoringCollectionConfig } from './config'; import { registerDynamicRoute } from './routes'; import { TYPE_ALLOWLIST } from './constants'; @@ -80,9 +80,9 @@ export class MonitoringCollectionPlugin implements Plugin Date: Wed, 8 Jun 2022 14:12:05 +0900 Subject: [PATCH 14/31] Switch to 0.29 and global metrics meter provider --- package.json | 9 +- .../server/monitoring/in_memory_metrics.ts | 31 ++- .../monitoring/register_node_collector.ts | 3 - .../monitoring_collection/server/config.ts | 1 + .../monitoring_collection/server/plugin.ts | 40 ++-- yarn.lock | 211 +++++++++--------- 6 files changed, 148 insertions(+), 147 deletions(-) diff --git a/package.json b/package.json index 37d83fa24566a..a906590648145 100644 --- a/package.json +++ b/package.json @@ -218,10 +218,11 @@ "@mapbox/mapbox-gl-draw": "1.3.0", "@mapbox/mapbox-gl-rtl-text": "0.2.3", "@mapbox/vector-tile": "1.3.1", - "@opentelemetry/api-metrics": "0.27.0", - "@opentelemetry/exporter-metrics-otlp-grpc": "^0.27.0", - "@opentelemetry/resources": "^1.0.1", - "@opentelemetry/sdk-metrics-base": "^0.27.0", + "@opentelemetry/api-metrics": "0.29.2", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.29.2", + "@opentelemetry/resources": "^1.3.1", + "@opentelemetry/sdk-metrics-base": "^0.29.2", + "@opentelemetry/semantic-conventions": "^1.3.1", "@reduxjs/toolkit": "^1.6.1", "@slack/webhook": "^5.0.4", "@turf/along": "6.0.1", diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index 31e51995dad7f..b7d8acfc5bbd2 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -6,7 +6,7 @@ */ import { Logger } from '@kbn/logging'; -import { Attributes, Counter, Meter } from '@opentelemetry/api-metrics'; +import { MetricAttributes, Counter, Meter, metrics } from '@opentelemetry/api-metrics'; export enum IN_MEMORY_METRICS { RULE_EXECUTIONS = 'ruleExecutions', @@ -22,17 +22,22 @@ export class InMemoryMetrics { [IN_MEMORY_METRICS.RULE_TIMEOUTS]: 0, }; - private otelMetrics?: { - [IN_MEMORY_METRICS.RULE_EXECUTIONS]: Counter; - [IN_MEMORY_METRICS.RULE_FAILURES]: Counter; - [IN_MEMORY_METRICS.RULE_TIMEOUTS]: Counter; - }; + private readonly meter: Meter; + private readonly metrics: Record; constructor(logger: Logger) { this.logger = logger; + this.meter = metrics.getMeter('kibana.alerting'); + this.metrics = { + [IN_MEMORY_METRICS.RULE_EXECUTIONS]: this.meter.createCounter( + IN_MEMORY_METRICS.RULE_EXECUTIONS + ), + [IN_MEMORY_METRICS.RULE_FAILURES]: this.meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), + [IN_MEMORY_METRICS.RULE_TIMEOUTS]: this.meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), + }; } - public increment(metric: IN_MEMORY_METRICS, attributes?: Attributes) { + public increment(metric: IN_MEMORY_METRICS, attributes?: MetricAttributes) { if (this.inMemoryMetrics[metric] === null) { this.logger.info( `Metric ${metric} is null because the counter ran over the max safe integer value, skipping increment.` @@ -49,9 +54,7 @@ export class InMemoryMetrics { (this.inMemoryMetrics[metric] as number)++; } - if (this.otelMetrics) { - this.otelMetrics[metric].add(1, attributes); - } + this.metrics[metric].add(1, attributes); } public getInMemoryMetric(metric: IN_MEMORY_METRICS) { @@ -61,12 +64,4 @@ export class InMemoryMetrics { public getAllInMemoryMetrics() { return this.inMemoryMetrics; } - - public registerMeter(meter: Meter) { - this.otelMetrics = { - [IN_MEMORY_METRICS.RULE_EXECUTIONS]: meter.createCounter(IN_MEMORY_METRICS.RULE_EXECUTIONS), - [IN_MEMORY_METRICS.RULE_FAILURES]: meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), - [IN_MEMORY_METRICS.RULE_TIMEOUTS]: meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), - }; - } } diff --git a/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts b/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts index 20e8e83836576..839787c6e78fc 100644 --- a/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts +++ b/x-pack/plugins/alerting/server/monitoring/register_node_collector.ts @@ -15,9 +15,6 @@ export function registerNodeCollector({ monitoringCollection: MonitoringCollectionSetup; inMemoryMetrics: InMemoryMetrics; }) { - const meter = monitoringCollection.getMeter('alerting-meter'); - inMemoryMetrics.registerMeter(meter); - monitoringCollection.registerMetric({ type: 'node_rules', schema: { diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index 3a9952cba12a7..1bdca7a6c2381 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -14,6 +14,7 @@ export const configSchema = schema.object({ otlp: schema.object({ url: schema.maybe(schema.string()), }), + exportIntervalMillis: schema.number({ defaultValue: 10000 }), }), }), }); diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 33b36293fa728..c498581fadcdd 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -9,17 +9,17 @@ import { JsonObject } from '@kbn/utility-types'; import { CoreSetup, Plugin, PluginInitializerContext, Logger } from '@kbn/core/server'; import { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server'; import { ServiceStatus } from '@kbn/core/server'; -import { Meter } from '@opentelemetry/api-metrics'; +import { metrics } from '@opentelemetry/api-metrics'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; -import { MeterProvider } from '@opentelemetry/sdk-metrics-base'; -// import { Resource } from '@opentelemetry/resources'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics-base'; +import { Resource } from '@opentelemetry/resources'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { MonitoringCollectionConfig } from './config'; import { registerDynamicRoute } from './routes'; import { TYPE_ALLOWLIST } from './constants'; export interface MonitoringCollectionSetup { registerMetric: (metric: Metric) => void; - getMeter: (name: string) => Meter; } export type MetricResult = T & JsonObject; @@ -75,18 +75,25 @@ export class MonitoringCollectionPlugin implements Plugin Date: Wed, 8 Jun 2022 16:15:00 +0900 Subject: [PATCH 15/31] Use http/s to toggle grpc secure handling --- package.json | 1 + .../monitoring_collection/server/plugin.ts | 50 +++++++++++-------- yarn.lock | 2 +- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index a906590648145..31eacef16abdd 100644 --- a/package.json +++ b/package.json @@ -122,6 +122,7 @@ "@emotion/css": "^11.9.0", "@emotion/react": "^11.9.0", "@emotion/serialize": "^1.0.3", + "@grpc/grpc-js": "^1.6.7", "@hapi/accept": "^5.0.2", "@hapi/boom": "^9.1.4", "@hapi/cookie": "^11.0.2", diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index c498581fadcdd..f99b6afef1fe9 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -14,6 +14,7 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics-base'; import { Resource } from '@opentelemetry/resources'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import * as grpc from '@grpc/grpc-js'; import { MonitoringCollectionConfig } from './config'; import { registerDynamicRoute } from './routes'; import { TYPE_ALLOWLIST } from './constants'; @@ -75,26 +76,7 @@ export class MonitoringCollectionPlugin implements Plugin(metric: Metric) => { @@ -115,6 +97,34 @@ export class MonitoringCollectionPlugin implements Plugin Date: Wed, 8 Jun 2022 16:37:21 +0900 Subject: [PATCH 16/31] Remove unnecesary getMeter method --- x-pack/plugins/monitoring_collection/server/mocks.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/monitoring_collection/server/mocks.ts b/x-pack/plugins/monitoring_collection/server/mocks.ts index a0cb9cb1d6fe7..9858653df648f 100644 --- a/x-pack/plugins/monitoring_collection/server/mocks.ts +++ b/x-pack/plugins/monitoring_collection/server/mocks.ts @@ -11,7 +11,6 @@ const createSetupMock = (): jest.Mocked => { const mock = { registerMetric: jest.fn(), getMetrics: jest.fn(), - getMeter: jest.fn(), }; return mock; }; From b796dd396311e618c65a81e0f4a382fef56afd48 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Wed, 8 Jun 2022 16:38:32 +0900 Subject: [PATCH 17/31] Remove unnecessary unused meterProvider member --- x-pack/plugins/monitoring_collection/server/plugin.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index f99b6afef1fe9..eb74b642e1e31 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -35,7 +35,6 @@ export class MonitoringCollectionPlugin implements Plugin> = {}; @@ -43,6 +42,7 @@ export class MonitoringCollectionPlugin implements Plugin Date: Wed, 8 Jun 2022 16:45:25 +0900 Subject: [PATCH 18/31] Avoid double grpcjs versions --- package.json | 2 +- yarn.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 31eacef16abdd..ec06205966487 100644 --- a/package.json +++ b/package.json @@ -122,7 +122,7 @@ "@emotion/css": "^11.9.0", "@emotion/react": "^11.9.0", "@emotion/serialize": "^1.0.3", - "@grpc/grpc-js": "^1.6.7", + "@grpc/grpc-js": "^1.5.9", "@hapi/accept": "^5.0.2", "@hapi/boom": "^9.1.4", "@hapi/cookie": "^11.0.2", diff --git a/yarn.lock b/yarn.lock index c0fe52644707b..f3406e8b2c4be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2001,7 +2001,7 @@ resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== -"@grpc/grpc-js@^1.5.9", "@grpc/grpc-js@^1.6.7": +"@grpc/grpc-js@^1.5.9": version "1.6.7" resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.6.7.tgz#4c4fa998ff719fe859ac19fe977fdef097bb99aa" integrity sha512-eBM03pu9hd3VqDQG+kHahiG1x80RGkkqqRb1Pchcwqej/KkAH95gAvKs6laqaHCycYaPK+TKuNQnOz9UXYA8qw== From ec4974251b84d2d9b27e003ac516bec67b2aeddc Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Wed, 8 Jun 2022 16:53:21 +0900 Subject: [PATCH 19/31] Update tests for new SDK & global meter provider --- x-pack/plugins/monitoring_collection/jest.config.js | 2 +- x-pack/plugins/monitoring_collection/server/plugin.test.ts | 5 ----- x-pack/plugins/monitoring_collection/server/plugin.ts | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/jest.config.js b/x-pack/plugins/monitoring_collection/jest.config.js index 1421a22541184..d2bb0617857d2 100644 --- a/x-pack/plugins/monitoring_collection/jest.config.js +++ b/x-pack/plugins/monitoring_collection/jest.config.js @@ -6,7 +6,7 @@ */ module.exports = { - preset: '@kbn/test/jest_node', + preset: '@kbn/test', rootDir: '../../..', roots: ['/x-pack/plugins/monitoring_collection'], coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/monitoring_collection', diff --git a/x-pack/plugins/monitoring_collection/server/plugin.test.ts b/x-pack/plugins/monitoring_collection/server/plugin.test.ts index d3e231cb7db83..15b9934525b9e 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.test.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.test.ts @@ -124,10 +124,5 @@ describe('monitoring_collection plugin', () => { ); expect(metrics).toBeUndefined(); }); - - it('should provide an opentelemetry meter', async () => { - const { getMeter } = plugin.setup(coreSetup); - expect(getMeter('monitoring-collection-plugin-test')).toBeDefined(); - }); }); }); diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index eb74b642e1e31..8572a5b7895c0 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -106,7 +106,7 @@ export class MonitoringCollectionPlugin implements Plugin Date: Wed, 8 Jun 2022 16:56:33 +0900 Subject: [PATCH 20/31] Remove extra config call --- x-pack/plugins/monitoring_collection/server/plugin.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 8572a5b7895c0..79738d07da73c 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -76,8 +76,6 @@ export class MonitoringCollectionPlugin implements Plugin(metric: Metric) => { if (this.metrics.hasOwnProperty(metric.type)) { From 10b63f00472eca012135f57c3fc5a861ec6aac47 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 9 Jun 2022 15:23:23 +0900 Subject: [PATCH 21/31] Add prometheus exporter --- package.json | 1 + yarn.lock | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/package.json b/package.json index f2c90cc971cb2..9db68ece73dcc 100644 --- a/package.json +++ b/package.json @@ -228,6 +228,7 @@ "@mapbox/vector-tile": "1.3.1", "@opentelemetry/api-metrics": "0.29.2", "@opentelemetry/exporter-metrics-otlp-grpc": "^0.29.2", + "@opentelemetry/exporter-prometheus": "^0.29.2", "@opentelemetry/resources": "^1.3.1", "@opentelemetry/sdk-metrics-base": "^0.29.2", "@opentelemetry/semantic-conventions": "^1.3.1", diff --git a/yarn.lock b/yarn.lock index f97bbd8307947..1b5154094140e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4200,6 +4200,15 @@ "@opentelemetry/resources" "1.3.1" "@opentelemetry/sdk-metrics-base" "0.29.2" +"@opentelemetry/exporter-prometheus@^0.29.2": + version "0.29.2" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.29.2.tgz#70ca7fb37655ca57a580387607d5465b47e27ac3" + integrity sha512-E5sRfUM4rzbvjxdpL1H6YRtjr8wY8+/2R4NjfxPEwrENLeeQk87V1E+YFLqAS7TfFLW7Zr4lmmamunwn5THvQA== + dependencies: + "@opentelemetry/api-metrics" "0.29.2" + "@opentelemetry/core" "1.3.1" + "@opentelemetry/sdk-metrics-base" "0.29.2" + "@opentelemetry/otlp-exporter-base@0.29.2": version "0.29.2" resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.29.2.tgz#fc844cc00e1ad95444b85056c08b1ca3e8a40447" From df02d12eb421a2cd8dbc3d65bf13e7e191cc1e11 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 9 Jun 2022 15:23:53 +0900 Subject: [PATCH 22/31] Wire up prometheus exporter when configured --- .../monitoring_collection/server/config.ts | 8 +++++++- .../monitoring_collection/server/plugin.ts | 20 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index 1bdca7a6c2381..cdcc83bfb00c6 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -13,8 +13,14 @@ export const configSchema = schema.object({ metrics: schema.object({ otlp: schema.object({ url: schema.maybe(schema.string()), + exportIntervalMillis: schema.number({ defaultValue: 10000 }), + }), + prometheus: schema.object({ + enabled: schema.boolean({ defaultValue: false }), + host: schema.string({ defaultValue: 'localhost' }), + port: schema.number({ defaultValue: 9464 }), + endpoint: schema.string({ defaultValue: '/metrics' }), }), - exportIntervalMillis: schema.number({ defaultValue: 10000 }), }), }), }); diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 79738d07da73c..1306bcafb9792 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -15,6 +15,7 @@ import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk import { Resource } from '@opentelemetry/resources'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import * as grpc from '@grpc/grpc-js'; +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; import { MonitoringCollectionConfig } from './config'; import { registerDynamicRoute } from './routes'; import { TYPE_ALLOWLIST } from './constants'; @@ -104,8 +105,9 @@ export class MonitoringCollectionPlugin implements Plugin Date: Thu, 9 Jun 2022 16:13:07 +0900 Subject: [PATCH 23/31] Fix unconfigured case --- x-pack/plugins/monitoring_collection/server/plugin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 1306bcafb9792..c72f2272f82cc 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -106,7 +106,7 @@ export class MonitoringCollectionPlugin implements Plugin Date: Thu, 9 Jun 2022 23:05:01 +0900 Subject: [PATCH 24/31] Move opentelemetry integration up to plugin level This avoids being entangled with the existing InMemoryMetrics implementation and uses opentelemetry directly. We have to update constructor signatures to pass the plugin metrics along side the InMemoryMetrics object that does the same. --- .../server/monitoring/in_memory_metrics.ts | 16 +-- .../alerting/server/monitoring/index.ts | 1 + .../server/monitoring/metrics.mock.ts | 18 +++ .../alerting/server/monitoring/metrics.ts | 16 +++ x-pack/plugins/alerting/server/plugin.ts | 11 +- .../alerting/server/rule_type_registry.ts | 8 +- .../server/task_runner/task_runner.test.ts | 134 ++++++++++++------ .../server/task_runner/task_runner.ts | 9 +- .../server/task_runner/task_runner_factory.ts | 7 +- 9 files changed, 153 insertions(+), 67 deletions(-) create mode 100644 x-pack/plugins/alerting/server/monitoring/metrics.mock.ts create mode 100644 x-pack/plugins/alerting/server/monitoring/metrics.ts diff --git a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts index b7d8acfc5bbd2..a2d0425da1427 100644 --- a/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/in_memory_metrics.ts @@ -6,7 +6,6 @@ */ import { Logger } from '@kbn/logging'; -import { MetricAttributes, Counter, Meter, metrics } from '@opentelemetry/api-metrics'; export enum IN_MEMORY_METRICS { RULE_EXECUTIONS = 'ruleExecutions', @@ -22,22 +21,11 @@ export class InMemoryMetrics { [IN_MEMORY_METRICS.RULE_TIMEOUTS]: 0, }; - private readonly meter: Meter; - private readonly metrics: Record; - constructor(logger: Logger) { this.logger = logger; - this.meter = metrics.getMeter('kibana.alerting'); - this.metrics = { - [IN_MEMORY_METRICS.RULE_EXECUTIONS]: this.meter.createCounter( - IN_MEMORY_METRICS.RULE_EXECUTIONS - ), - [IN_MEMORY_METRICS.RULE_FAILURES]: this.meter.createCounter(IN_MEMORY_METRICS.RULE_FAILURES), - [IN_MEMORY_METRICS.RULE_TIMEOUTS]: this.meter.createCounter(IN_MEMORY_METRICS.RULE_TIMEOUTS), - }; } - public increment(metric: IN_MEMORY_METRICS, attributes?: MetricAttributes) { + public increment(metric: IN_MEMORY_METRICS) { if (this.inMemoryMetrics[metric] === null) { this.logger.info( `Metric ${metric} is null because the counter ran over the max safe integer value, skipping increment.` @@ -53,8 +41,6 @@ export class InMemoryMetrics { } else { (this.inMemoryMetrics[metric] as number)++; } - - this.metrics[metric].add(1, attributes); } public getInMemoryMetric(metric: IN_MEMORY_METRICS) { diff --git a/x-pack/plugins/alerting/server/monitoring/index.ts b/x-pack/plugins/alerting/server/monitoring/index.ts index 5f298456554f0..3040758f6a4d7 100644 --- a/x-pack/plugins/alerting/server/monitoring/index.ts +++ b/x-pack/plugins/alerting/server/monitoring/index.ts @@ -8,3 +8,4 @@ export { registerNodeCollector } from './register_node_collector'; export { registerClusterCollector } from './register_cluster_collector'; export * from './types'; export * from './in_memory_metrics'; +export * from './metrics'; diff --git a/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts b/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts new file mode 100644 index 0000000000000..280acabac2329 --- /dev/null +++ b/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +function createInMemoryMetricsMock() { + return jest.fn().mockImplementation(() => { + return { + ruleExecutions: { add: jest.fn() }, + }; + }); +} + +export const metricsMock = { + create: createInMemoryMetricsMock(), +}; diff --git a/x-pack/plugins/alerting/server/monitoring/metrics.ts b/x-pack/plugins/alerting/server/monitoring/metrics.ts new file mode 100644 index 0000000000000..be566711252f5 --- /dev/null +++ b/x-pack/plugins/alerting/server/monitoring/metrics.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Counter, Meter } from '@opentelemetry/api-metrics'; + +export class Metrics { + ruleExecutions: Counter; + + constructor(meter: Meter) { + this.ruleExecutions = meter.createCounter('ruleExecutions'); + } +} diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index a3b714624fbf3..2851ed8356c49 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -49,6 +49,7 @@ import { import { PluginStartContract as FeaturesPluginStart } from '@kbn/features-plugin/server'; import { PluginStart as DataPluginStart } from '@kbn/data-plugin/server'; import { MonitoringCollectionSetup } from '@kbn/monitoring-collection-plugin/server'; +import { metrics } from '@opentelemetry/api-metrics'; import { RulesClient } from './rules_client'; import { RuleTypeRegistry } from './rule_type_registry'; import { TaskRunnerFactory } from './task_runner'; @@ -77,7 +78,12 @@ import { getHealth } from './health/get_health'; import { AlertingAuthorizationClientFactory } from './alerting_authorization_client_factory'; import { AlertingAuthorization } from './authorization'; import { getSecurityHealth, SecurityHealth } from './lib/get_security_health'; -import { registerNodeCollector, registerClusterCollector, InMemoryMetrics } from './monitoring'; +import { + registerNodeCollector, + registerClusterCollector, + InMemoryMetrics, + Metrics, +} from './monitoring'; import { getRuleTaskTimeout } from './lib/get_rule_task_timeout'; import { getActionsConfigMap } from './lib/get_actions_config_map'; @@ -173,6 +179,7 @@ export class AlertingPlugin { private kibanaBaseUrl: string | undefined; private usageCounter: UsageCounter | undefined; private inMemoryMetrics: InMemoryMetrics; + private metrics: Metrics; constructor(initializerContext: PluginInitializerContext) { this.config = initializerContext.config.get(); @@ -183,6 +190,7 @@ export class AlertingPlugin { this.telemetryLogger = initializerContext.logger.get('usage'); this.kibanaVersion = initializerContext.env.packageInfo.version; this.inMemoryMetrics = new InMemoryMetrics(initializerContext.logger.get('in_memory_metrics')); + this.metrics = new Metrics(metrics.getMeter('kibana.alerting')); } public setup( @@ -227,6 +235,7 @@ export class AlertingPlugin { licensing: plugins.licensing, minimumScheduleInterval: this.config.rules.minimumScheduleInterval, inMemoryMetrics: this.inMemoryMetrics, + metrics: this.metrics, }); this.ruleTypeRegistry = ruleTypeRegistry; diff --git a/x-pack/plugins/alerting/server/rule_type_registry.ts b/x-pack/plugins/alerting/server/rule_type_registry.ts index 338450746781b..f6a164b423727 100644 --- a/x-pack/plugins/alerting/server/rule_type_registry.ts +++ b/x-pack/plugins/alerting/server/rule_type_registry.ts @@ -31,7 +31,7 @@ import { } from '../common'; import { ILicenseState } from './lib/license_state'; import { getRuleTypeFeatureUsageName } from './lib/get_rule_type_feature_usage_name'; -import { InMemoryMetrics } from './monitoring'; +import { InMemoryMetrics, Metrics } from './monitoring'; import { AlertingRulesConfig } from '.'; export interface ConstructorOptions { @@ -42,6 +42,7 @@ export interface ConstructorOptions { licensing: LicensingPluginSetup; minimumScheduleInterval: AlertingRulesConfig['minimumScheduleInterval']; inMemoryMetrics: InMemoryMetrics; + metrics: Metrics; } export interface RegistryRuleType @@ -139,6 +140,7 @@ export class RuleTypeRegistry { private readonly minimumScheduleInterval: AlertingRulesConfig['minimumScheduleInterval']; private readonly licensing: LicensingPluginSetup; private readonly inMemoryMetrics: InMemoryMetrics; + private readonly metrics: Metrics; constructor({ logger, @@ -148,6 +150,7 @@ export class RuleTypeRegistry { licensing, minimumScheduleInterval, inMemoryMetrics, + metrics, }: ConstructorOptions) { this.logger = logger; this.taskManager = taskManager; @@ -156,6 +159,7 @@ export class RuleTypeRegistry { this.licensing = licensing; this.minimumScheduleInterval = minimumScheduleInterval; this.inMemoryMetrics = inMemoryMetrics; + this.metrics = metrics; } public has(id: string) { @@ -274,7 +278,7 @@ export class RuleTypeRegistry { InstanceContext, ActionGroupIds, RecoveryActionGroupId | RecoveredActionGroupId - >(normalizedRuleType, context, this.inMemoryMetrics), + >(normalizedRuleType, context, this.inMemoryMetrics, this.metrics), }, }); // No need to notify usage on basic alert types diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index f3d2c7039585b..119b4f9c7c1a8 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -43,6 +43,7 @@ import { omit } from 'lodash'; import { ruleTypeRegistryMock } from '../rule_type_registry.mock'; import { ExecuteOptions } from '@kbn/actions-plugin/server/create_execute_function'; import { inMemoryMetricsMock } from '../monitoring/in_memory_metrics.mock'; +import { metricsMock } from '../monitoring/metrics.mock'; import moment from 'moment'; import { generateAlertOpts, @@ -122,6 +123,7 @@ describe('Task Runner', () => { const dataPlugin = dataPluginMock.createStartContract(); const uiSettingsService = uiSettingsServiceMock.createStartContract(); const inMemoryMetrics = inMemoryMetricsMock.create(); + const metrics = metricsMock.create(); type TaskRunnerFactoryInitializerParamsType = jest.Mocked & { actionsPlugin: jest.Mocked; @@ -219,7 +221,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -321,7 +324,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -403,7 +407,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -525,7 +530,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -579,7 +585,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -659,7 +666,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -704,7 +712,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -762,7 +771,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -830,7 +840,8 @@ describe('Task Runner', () => { }, }, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -911,7 +922,8 @@ describe('Task Runner', () => { }, }, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -976,7 +988,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1088,7 +1101,8 @@ describe('Task Runner', () => { }, }, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1201,7 +1215,8 @@ describe('Task Runner', () => { }, }, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1293,7 +1308,8 @@ describe('Task Runner', () => { }, }, customTaskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1384,7 +1400,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1447,7 +1464,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1466,7 +1484,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1496,7 +1515,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1528,7 +1548,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1565,7 +1586,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1594,7 +1616,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1623,7 +1646,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1652,7 +1676,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1682,7 +1707,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1715,7 +1741,8 @@ describe('Task Runner', () => { ruleType, legacyTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1753,7 +1780,8 @@ describe('Task Runner', () => { state: originalAlertSate, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1783,7 +1811,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1814,7 +1843,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1838,7 +1868,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1864,7 +1895,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -1912,7 +1944,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2018,7 +2051,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2092,7 +2126,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2161,7 +2196,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2235,7 +2271,8 @@ describe('Task Runner', () => { }, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2287,7 +2324,8 @@ describe('Task Runner', () => { ...taskRunnerFactoryInitializerParams, supportsEphemeralTasks: true, }, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2359,7 +2397,8 @@ describe('Task Runner', () => { state, }, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2388,7 +2427,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2403,7 +2443,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2431,7 +2472,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2466,7 +2508,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2551,7 +2594,8 @@ describe('Task Runner', () => { ...taskRunnerFactoryInitializerParams, actionsConfigMap, }, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2704,7 +2748,8 @@ describe('Task Runner', () => { ...taskRunnerFactoryInitializerParams, actionsConfigMap, }, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); @@ -2775,7 +2820,8 @@ describe('Task Runner', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalled(); diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index bd83b269ce10d..fc09e7c80802b 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -57,7 +57,7 @@ import { } from '../../common'; import { NormalizedRuleType, UntypedNormalizedRuleType } from '../rule_type_registry'; import { getEsErrorMessage } from '../lib/errors'; -import { InMemoryMetrics, IN_MEMORY_METRICS } from '../monitoring'; +import { InMemoryMetrics, IN_MEMORY_METRICS, Metrics } from '../monitoring'; import { GenerateNewAndRecoveredAlertEventsParams, LogActiveAndRecoveredAlertsParams, @@ -112,6 +112,7 @@ export class TaskRunner< private readonly executionId: string; private readonly ruleTypeRegistry: RuleTypeRegistry; private readonly inMemoryMetrics: InMemoryMetrics; + private readonly metrics: Metrics; private alertingEventLogger: AlertingEventLogger; private usageCounter?: UsageCounter; private searchAbortController: AbortController; @@ -129,7 +130,8 @@ export class TaskRunner< >, taskInstance: ConcreteTaskInstance, context: TaskRunnerContext, - inMemoryMetrics: InMemoryMetrics + inMemoryMetrics: InMemoryMetrics, + metrics: Metrics ) { this.context = context; this.logger = context.logger; @@ -142,6 +144,7 @@ export class TaskRunner< this.cancelled = false; this.executionId = uuid.v4(); this.inMemoryMetrics = inMemoryMetrics; + this.metrics = metrics; this.alertingEventLogger = new AlertingEventLogger(this.context.eventLogger); } @@ -798,6 +801,8 @@ export class TaskRunner< if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); + // NOTE: Using the typesafe opentelemetry counter here directly + this.metrics.ruleExecutions.add(1); if (executionStatus.error) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.ts index e7c483b944ed1..135886121d25c 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.ts @@ -33,7 +33,7 @@ import { import { TaskRunner } from './task_runner'; import { RulesClient } from '../rules_client'; import { NormalizedRuleType } from '../rule_type_registry'; -import { InMemoryMetrics } from '../monitoring'; +import { InMemoryMetrics, Metrics } from '../monitoring'; import { ActionsConfigMap } from '../lib/get_actions_config_map'; export interface TaskRunnerContext { @@ -90,7 +90,8 @@ export class TaskRunnerFactory { RecoveryActionGroupId >, { taskInstance }: RunContext, - inMemoryMetrics: InMemoryMetrics + inMemoryMetrics: InMemoryMetrics, + metrics: Metrics ) { if (!this.isInitialized) { throw new Error('TaskRunnerFactory not initialized'); @@ -104,6 +105,6 @@ export class TaskRunnerFactory { InstanceContext, ActionGroupIds, RecoveryActionGroupId - >(ruleType, taskInstance, this.taskRunnerContext!, inMemoryMetrics); + >(ruleType, taskInstance, this.taskRunnerContext!, inMemoryMetrics, metrics); } } From 339951f6d90079ea779eb75dddd5bd71d4d0e9af Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 13 Jun 2022 12:08:23 +0900 Subject: [PATCH 25/31] Add missing test mocks --- .../alerting/server/rule_type_registry.test.ts | 3 +++ .../saved_objects/is_rule_exportable.test.ts | 3 +++ .../server/task_runner/task_runner_cancel.test.ts | 14 ++++++++++---- .../server/task_runner/task_runner_factory.test.ts | 4 +++- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/alerting/server/rule_type_registry.test.ts b/x-pack/plugins/alerting/server/rule_type_registry.test.ts index 9a6b2232c47d4..8b96d1e706c70 100644 --- a/x-pack/plugins/alerting/server/rule_type_registry.test.ts +++ b/x-pack/plugins/alerting/server/rule_type_registry.test.ts @@ -14,6 +14,7 @@ import { licenseStateMock } from './lib/license_state.mock'; import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { loggingSystemMock } from '@kbn/core/server/mocks'; import { inMemoryMetricsMock } from './monitoring/in_memory_metrics.mock'; +import { metricsMock } from './monitoring/metrics.mock'; const logger = loggingSystemMock.create().get(); let mockedLicenseState: jest.Mocked; @@ -22,6 +23,7 @@ let ruleTypeRegistryParams: ConstructorOptions; const taskManager = taskManagerMock.createSetup(); const inMemoryMetrics = inMemoryMetricsMock.create(); +const metrics = metricsMock.create(); beforeEach(() => { jest.resetAllMocks(); @@ -34,6 +36,7 @@ beforeEach(() => { licensing: licensingMock.createSetup(), minimumScheduleInterval: { value: '1m', enforce: false }, inMemoryMetrics, + metrics, }; }); diff --git a/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts b/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts index 0f2677dc49751..81992e1a50510 100644 --- a/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts +++ b/x-pack/plugins/alerting/server/saved_objects/is_rule_exportable.test.ts @@ -15,12 +15,14 @@ import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { isRuleExportable } from './is_rule_exportable'; import { inMemoryMetricsMock } from '../monitoring/in_memory_metrics.mock'; import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { metricsMock } from '../monitoring/metrics.mock'; let ruleTypeRegistryParams: ConstructorOptions; let logger: MockedLogger; let mockedLicenseState: jest.Mocked; const taskManager = taskManagerMock.createSetup(); const inMemoryMetrics = inMemoryMetricsMock.create(); +const metrics = metricsMock.create(); beforeEach(() => { jest.resetAllMocks(); @@ -34,6 +36,7 @@ beforeEach(() => { licensing: licensingMock.createSetup(), minimumScheduleInterval: { value: '1m', enforce: false }, inMemoryMetrics, + metrics, }; }); diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts index fb2d1be3a3872..f2d4578689f32 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts @@ -35,6 +35,7 @@ import { IEventLogger } from '@kbn/event-log-plugin/server'; import { ruleTypeRegistryMock } from '../rule_type_registry.mock'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { inMemoryMetricsMock } from '../monitoring/in_memory_metrics.mock'; +import { metricsMock } from '../monitoring/metrics.mock'; import { AlertingEventLogger, RuleContextOpts, @@ -95,6 +96,7 @@ describe('Task Runner Cancel', () => { const uiSettingsService = uiSettingsServiceMock.createStartContract(); const dataPlugin = dataPluginMock.createStartContract(); const inMemoryMetrics = inMemoryMetricsMock.create(); + const metrics = metricsMock.create(); type TaskRunnerFactoryInitializerParamsType = jest.Mocked & { actionsPlugin: jest.Mocked; @@ -176,7 +178,8 @@ describe('Task Runner Cancel', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -244,7 +247,8 @@ describe('Task Runner Cancel', () => { ...taskRunnerFactoryInitializerParams, cancelAlertsOnRuleTimeout: false, }, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -308,7 +312,8 @@ describe('Task Runner Cancel', () => { updatedRuleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); @@ -368,7 +373,8 @@ describe('Task Runner Cancel', () => { ruleType, mockedTaskInstance, taskRunnerFactoryInitializerParams, - inMemoryMetrics + inMemoryMetrics, + metrics ); expect(AlertingEventLogger).toHaveBeenCalledTimes(1); diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts index e787617800356..ea6c1a0c63d81 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts @@ -26,8 +26,10 @@ import { ruleTypeRegistryMock } from '../rule_type_registry.mock'; import { executionContextServiceMock } from '@kbn/core/server/mocks'; import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { inMemoryMetricsMock } from '../monitoring/in_memory_metrics.mock'; +import { metricsMock } from '../monitoring/metrics.mock'; const inMemoryMetrics = inMemoryMetricsMock.create(); +const metrics = metricsMock.create(); const executionContext = executionContextServiceMock.createSetupContract(); const mockUsageCountersSetup = usageCountersServiceMock.createSetupContract(); const mockUsageCounter = mockUsageCountersSetup.createUsageCounter('test'); @@ -115,7 +117,7 @@ describe('Task Runner Factory', () => { test(`throws an error if factory isn't initialized`, () => { const factory = new TaskRunnerFactory(); expect(() => - factory.create(ruleType, { taskInstance: mockedTaskInstance }, inMemoryMetrics) + factory.create(ruleType, { taskInstance: mockedTaskInstance }, inMemoryMetrics, metrics) ).toThrowErrorMatchingInlineSnapshot(`"TaskRunnerFactory not initialized"`); }); From 05389c328500fd51430c0224dfe408b78bc1bbe4 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 13 Jun 2022 14:52:41 +0900 Subject: [PATCH 26/31] Add authenticated prometheus endpoint This was a little tricky due to how PrometheusExporter/PrometheusSerializer are implemented upstream. If we expose the serializer it gets simpler, since we can just do a simple Kibana-compatible PrometheusExporter that uses the same serializer. --- .../monitoring_collection/server/config.ts | 3 - .../monitoring_collection/server/constants.ts | 2 + .../server/lib/prometheus_exporter.ts | 75 +++++ .../server/lib/prometheus_serializer.ts | 310 ++++++++++++++++++ .../monitoring_collection/server/plugin.ts | 28 +- .../server/routes/index.ts | 1 + .../server/routes/prometheus.test.ts | 29 ++ .../server/routes/prometheus.ts | 32 ++ 8 files changed, 465 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts create mode 100644 x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts create mode 100644 x-pack/plugins/monitoring_collection/server/routes/prometheus.test.ts create mode 100644 x-pack/plugins/monitoring_collection/server/routes/prometheus.ts diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index cdcc83bfb00c6..f6b5a62925d7e 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -17,9 +17,6 @@ export const configSchema = schema.object({ }), prometheus: schema.object({ enabled: schema.boolean({ defaultValue: false }), - host: schema.string({ defaultValue: 'localhost' }), - port: schema.number({ defaultValue: 9464 }), - endpoint: schema.string({ defaultValue: '/metrics' }), }), }), }), diff --git a/x-pack/plugins/monitoring_collection/server/constants.ts b/x-pack/plugins/monitoring_collection/server/constants.ts index 86231dec6c6c2..4a698aa26f29b 100644 --- a/x-pack/plugins/monitoring_collection/server/constants.ts +++ b/x-pack/plugins/monitoring_collection/server/constants.ts @@ -5,3 +5,5 @@ * 2.0. */ export const TYPE_ALLOWLIST = ['node_rules', 'cluster_rules', 'node_actions', 'cluster_actions']; + +export const PROMETHEUS_ROUTE = '/api/monitoring_collection/v1/prometheus'; diff --git a/x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts b/x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts new file mode 100644 index 0000000000000..d4b21fbd9641f --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/lib/prometheus_exporter.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Adapted from https://github.com/open-telemetry/opentelemetry-js/blob/aabc5f6b89e3d9af6640fb854967212ca5b1a3b8/experimental/packages/opentelemetry-exporter-prometheus/src/PrometheusExporter.ts + +import { AggregationTemporality, MetricReader } from '@opentelemetry/sdk-metrics-base'; +import { PrometheusExporter as OpenTelemetryPrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { ExporterConfig } from '@opentelemetry/exporter-prometheus'; +import { KibanaResponseFactory } from '@kbn/core/server'; +import { Logger } from '@kbn/core/server'; +import { PrometheusSerializer } from './prometheus_serializer'; + +export class PrometheusExporter extends MetricReader { + private readonly _prefix?: string; + private readonly _appendTimestamp: boolean; + private _serializer: PrometheusSerializer; + + constructor(logger: Logger, config: ExporterConfig = {}) { + super(); + this._prefix = config.prefix || OpenTelemetryPrometheusExporter.DEFAULT_OPTIONS.prefix; + this._appendTimestamp = + typeof config.appendTimestamp === 'boolean' + ? config.appendTimestamp + : OpenTelemetryPrometheusExporter.DEFAULT_OPTIONS.appendTimestamp; + + this._serializer = new PrometheusSerializer(logger, this._prefix, this._appendTimestamp); + } + + selectAggregationTemporality(): AggregationTemporality { + return AggregationTemporality.CUMULATIVE; + } + + protected onForceFlush(): Promise { + return Promise.resolve(undefined); + } + + protected onShutdown(): Promise { + return Promise.resolve(undefined); + } + + /** + * Responds to incoming message with current state of all metrics. + */ + public exportMetrics(res: KibanaResponseFactory) { + // TODO: How can I type this return without requiring (forbidden path) KibanaReponse? + return this.collect().then( + (collectionResult) => { + const { resourceMetrics, errors } = collectionResult; + if (errors.length) { + return res.customError({ + statusCode: 500, + body: `PrometheusExporter: metrics collection errors ${errors}`, + }); + } + const result = this._serializer.serialize(resourceMetrics); + if (result === '') { + return res.noContent(); + } + return res.ok({ + body: result, + }); + }, + (err) => { + return res.customError({ + statusCode: 500, + body: `# Failed to export metrics ${err}`, + }); + } + ); + } +} diff --git a/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts b/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts new file mode 100644 index 0000000000000..3adcffbaff20d --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts @@ -0,0 +1,310 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// From https://github.com/open-telemetry/opentelemetry-js/blob/97bc6321c0fe4da7414afb83038279b735a5ba65/experimental/packages/opentelemetry-exporter-prometheus/src/PrometheusSerializer.ts + +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from '@kbn/core/server'; +import { + ResourceMetrics, + InstrumentType, + DataPointType, + ScopeMetrics, + MetricData, + DataPoint, + Histogram, +} from '@opentelemetry/sdk-metrics-base'; +import type { MetricAttributes } from '@opentelemetry/api-metrics'; + +// From https://github.com/open-telemetry/opentelemetry-js/blob/28c9e8829488a7fa131803447b0511195ae1fdf0/packages/opentelemetry-core/src/common/time.ts#L148 +export function hrTimeToMilliseconds(time: [number, number]): number { + return Math.round(time[0] * 1e3 + time[1] / 1e6); +} + +type PrometheusDataTypeLiteral = 'counter' | 'gauge' | 'histogram' | 'summary' | 'untyped'; + +function escapeString(str: string) { + return str.replace(/\\/g, '\\\\').replace(/\n/g, '\\n'); +} + +function escapeAttributeValue(str: string) { + if (typeof str !== 'string') { + str = String(str); + } + return escapeString(str).replace(/"/g, '\\"'); +} + +const invalidCharacterRegex = /[^a-z0-9_]/gi; +/** + * Ensures metric names are valid Prometheus metric names by removing + * characters allowed by OpenTelemetry but disallowed by Prometheus. + * + * https://prometheus.io/docs/concepts/data_model/#metric-names-and-attributes + * + * 1. Names must match `[a-zA-Z_:][a-zA-Z0-9_:]*` + * + * 2. Colons are reserved for user defined recording rules. + * They should not be used by exporters or direct instrumentation. + * + * OpenTelemetry metric names are already validated in the Meter when they are created, + * and they match the format `[a-zA-Z][a-zA-Z0-9_.\-]*` which is very close to a valid + * prometheus metric name, so we only need to strip characters valid in OpenTelemetry + * but not valid in prometheus and replace them with '_'. + * + * @param name name to be sanitized + */ +function sanitizePrometheusMetricName(name: string): string { + return name.replace(invalidCharacterRegex, '_'); // replace all invalid characters with '_' +} + +/** + * @private + * + * Helper method which assists in enforcing the naming conventions for metric + * names in Prometheus + * @param name the name of the metric + * @param type the kind of metric + * @returns string + */ +function enforcePrometheusNamingConvention(name: string, type: InstrumentType): string { + // Prometheus requires that metrics of the Counter kind have "_total" suffix + if (!name.endsWith('_total') && type === InstrumentType.COUNTER) { + name = name + '_total'; + } + + return name; +} + +function valueString(value: number) { + if (Number.isNaN(value)) { + return 'Nan'; + } else if (!Number.isFinite(value)) { + if (value < 0) { + return '-Inf'; + } else { + return '+Inf'; + } + } else { + return `${value}`; + } +} + +function toPrometheusType( + instrumentType: InstrumentType, + dataPointType: DataPointType +): PrometheusDataTypeLiteral { + switch (dataPointType) { + case DataPointType.SINGULAR: + if ( + instrumentType === InstrumentType.COUNTER || + instrumentType === InstrumentType.OBSERVABLE_COUNTER + ) { + return 'counter'; + } + /** + * - HISTOGRAM + * - UP_DOWN_COUNTER + * - OBSERVABLE_GAUGE + * - OBSERVABLE_UP_DOWN_COUNTER + */ + return 'gauge'; + case DataPointType.HISTOGRAM: + return 'histogram'; + default: + return 'untyped'; + } +} + +function stringify( + metricName: string, + attributes: MetricAttributes, + value: number, + timestamp?: number, + additionalAttributes?: MetricAttributes +) { + let hasAttribute = false; + let attributesStr = ''; + + for (const [key, val] of Object.entries(attributes)) { + const sanitizedAttributeName = sanitizePrometheusMetricName(key); + hasAttribute = true; + attributesStr += `${ + attributesStr.length > 0 ? ',' : '' + }${sanitizedAttributeName}="${escapeAttributeValue(val)}"`; + } + if (additionalAttributes) { + for (const [key, val] of Object.entries(additionalAttributes)) { + const sanitizedAttributeName = sanitizePrometheusMetricName(key); + hasAttribute = true; + attributesStr += `${ + attributesStr.length > 0 ? ',' : '' + }${sanitizedAttributeName}="${escapeAttributeValue(val)}"`; + } + } + + if (hasAttribute) { + metricName += `{${attributesStr}}`; + } + + return `${metricName} ${valueString(value)}${ + timestamp !== undefined ? ' ' + String(timestamp) : '' + }\n`; +} + +export class PrometheusSerializer { + private logger: Logger; + private _prefix: string | undefined; + private _appendTimestamp: boolean; + + constructor(logger: Logger, prefix?: string, appendTimestamp = true) { + if (prefix) { + this._prefix = prefix + '_'; + } + this._appendTimestamp = appendTimestamp; + this.logger = logger; + } + + serialize(resourceMetrics: ResourceMetrics): string { + let str = ''; + for (const scopeMetrics of resourceMetrics.scopeMetrics) { + str += this.serializeScopeMetrics(scopeMetrics); + } + return str; + } + + serializeScopeMetrics(scopeMetrics: ScopeMetrics) { + let str = ''; + for (const metric of scopeMetrics.metrics) { + str += this.serializeMetricData(metric) + '\n'; + } + return str; + } + + serializeMetricData(metricData: MetricData) { + let name = sanitizePrometheusMetricName(escapeString(metricData.descriptor.name)); + if (this._prefix) { + name = `${this._prefix}${name}`; + } + const dataPointType = metricData.dataPointType; + + name = enforcePrometheusNamingConvention(name, metricData.descriptor.type); + + const help = `# HELP ${name} ${escapeString( + metricData.descriptor.description || 'description missing' + )}`; + const type = `# TYPE ${name} ${toPrometheusType(metricData.descriptor.type, dataPointType)}`; + + let results = ''; + switch (dataPointType) { + case DataPointType.SINGULAR: { + results = metricData.dataPoints + .map((it) => this.serializeSingularDataPoint(name, metricData.descriptor.type, it)) + .join(''); + break; + } + case DataPointType.HISTOGRAM: { + results = metricData.dataPoints + .map((it) => this.serializeHistogramDataPoint(name, metricData.descriptor.type, it)) + .join(''); + break; + } + default: { + this.logger.error(`Unrecognizable DataPointType: ${dataPointType} for metric "${name}"`); + } + } + + return `${help}\n${type}\n${results}`.trim(); + } + + serializeSingularDataPoint( + name: string, + type: InstrumentType, + dataPoint: DataPoint + ): string { + let results = ''; + + name = enforcePrometheusNamingConvention(name, type); + const { value, attributes } = dataPoint; + const timestamp = hrTimeToMilliseconds(dataPoint.endTime); + results += stringify( + name, + attributes, + value, + this._appendTimestamp ? timestamp : undefined, + undefined + ); + return results; + } + + serializeHistogramDataPoint( + name: string, + type: InstrumentType, + dataPoint: DataPoint + ): string { + let results = ''; + + name = enforcePrometheusNamingConvention(name, type); + const { value, attributes } = dataPoint; + const timestamp = hrTimeToMilliseconds(dataPoint.endTime); + /** Histogram["bucket"] is not typed with `number` */ + for (const key of ['count', 'sum'] as Array<'count' | 'sum'>) { + results += stringify( + name + '_' + key, + attributes, + value[key], + this._appendTimestamp ? timestamp : undefined, + undefined + ); + } + + let cumulativeSum = 0; + const countEntries = value.buckets.counts.entries(); + let infiniteBoundaryDefined = false; + for (const [idx, val] of countEntries) { + cumulativeSum += val; + const upperBound = value.buckets.boundaries[idx]; + /** HistogramAggregator is producing different boundary output - + * in one case not including infinity values, in other - + * full, e.g. [0, 100] and [0, 100, Infinity] + * we should consider that in export, if Infinity is defined, use it + * as boundary + */ + if (upperBound === undefined && infiniteBoundaryDefined) { + break; + } + if (upperBound === Infinity) { + infiniteBoundaryDefined = true; + } + results += stringify( + name + '_bucket', + attributes, + cumulativeSum, + this._appendTimestamp ? timestamp : undefined, + { + le: upperBound === undefined || upperBound === Infinity ? '+Inf' : String(upperBound), + } + ); + } + + return results; + } +} diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index c72f2272f82cc..2440fbddd6f8e 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -15,10 +15,10 @@ import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk import { Resource } from '@opentelemetry/resources'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import * as grpc from '@grpc/grpc-js'; -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { PrometheusExporter } from './lib/prometheus_exporter'; import { MonitoringCollectionConfig } from './config'; -import { registerDynamicRoute } from './routes'; -import { TYPE_ALLOWLIST } from './constants'; +import { registerDynamicRoute, registerV1PrometheusRoute } from './routes'; +import { PROMETHEUS_ROUTE, TYPE_ALLOWLIST } from './constants'; export interface MonitoringCollectionSetup { registerMetric: (metric: Metric) => void; @@ -39,6 +39,8 @@ export class MonitoringCollectionPlugin implements Plugin> = {}; + private prometheusExporter?: PrometheusExporter; + constructor(initializerContext: PluginInitializerContext) { this.initializerContext = initializerContext; this.logger = initializerContext.logger.get(); @@ -63,6 +65,10 @@ export class MonitoringCollectionPlugin implements Plugin { + it('forwards the request to the prometheus exporter', async () => { + const router = httpServiceMock.createRouter(); + const prometheusExporter = { exportMetrics: jest.fn() } as unknown as PrometheusExporter; + registerV1PrometheusRoute({ router, prometheusExporter }); + + const [, handler] = router.get.mock.calls[0]; + + const context = {}; + const req = { params: {} } as KibanaRequest; + const factory: jest.Mocked = httpServerMock.createResponseFactory(); + + await handler(context, req, factory); + + expect(prometheusExporter.exportMetrics).toHaveBeenCalledWith(factory); + }); +}); diff --git a/x-pack/plugins/monitoring_collection/server/routes/prometheus.ts b/x-pack/plugins/monitoring_collection/server/routes/prometheus.ts new file mode 100644 index 0000000000000..40e783c3d8b46 --- /dev/null +++ b/x-pack/plugins/monitoring_collection/server/routes/prometheus.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IRouter } from '@kbn/core/server'; +import { PROMETHEUS_ROUTE } from '../constants'; +import { PrometheusExporter } from '../lib/prometheus_exporter'; + +export function registerV1PrometheusRoute({ + router, + prometheusExporter, +}: { + router: IRouter; + prometheusExporter: PrometheusExporter; +}) { + router.get( + { + path: PROMETHEUS_ROUTE, + options: { + authRequired: true, + tags: ['api'], // ensures that unauthenticated calls receive a 401 rather than a 302 redirect to login page + }, + validate: {}, + }, + async (context, req, res) => { + return prometheusExporter.exportMetrics(res); + } + ); +} From a657820eeb00b08525eadd74be37e141084ac0b8 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 13 Jun 2022 15:10:49 +0900 Subject: [PATCH 27/31] Add note about upstream PR --- .../monitoring_collection/server/lib/prometheus_serializer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts b/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts index 3adcffbaff20d..d036a31673700 100644 --- a/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts +++ b/x-pack/plugins/monitoring_collection/server/lib/prometheus_serializer.ts @@ -6,6 +6,7 @@ */ // From https://github.com/open-telemetry/opentelemetry-js/blob/97bc6321c0fe4da7414afb83038279b735a5ba65/experimental/packages/opentelemetry-exporter-prometheus/src/PrometheusSerializer.ts +// Can be removed once https://github.com/open-telemetry/opentelemetry-js/issues/3033 is merged/released /* * Copyright The OpenTelemetry Authors From aee16d61286e2a9d78249905cc5dfa85c3b29b50 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Mon, 13 Jun 2022 22:46:23 +0900 Subject: [PATCH 28/31] Wire up additional otel attributes --- .../monitoring_collection/server/plugin.ts | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 2440fbddd6f8e..97c7136e2663a 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -45,7 +45,6 @@ export class MonitoringCollectionPlugin implements Plugin; core.status.overall$.subscribe((newStatus) => { @@ -73,9 +77,9 @@ export class MonitoringCollectionPlugin implements Plugin status, getMetric: async (type: string) => { @@ -102,10 +106,16 @@ export class MonitoringCollectionPlugin implements Plugin Date: Tue, 14 Jun 2022 11:38:21 +0900 Subject: [PATCH 29/31] Additional instrumentation --- x-pack/plugins/alerting/server/monitoring/metrics.ts | 8 +++++++- x-pack/plugins/alerting/server/task_runner/task_runner.ts | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/alerting/server/monitoring/metrics.ts b/x-pack/plugins/alerting/server/monitoring/metrics.ts index be566711252f5..b13fefa65b5c6 100644 --- a/x-pack/plugins/alerting/server/monitoring/metrics.ts +++ b/x-pack/plugins/alerting/server/monitoring/metrics.ts @@ -5,12 +5,18 @@ * 2.0. */ -import { Counter, Meter } from '@opentelemetry/api-metrics'; +import { Counter, Histogram, Meter } from '@opentelemetry/api-metrics'; export class Metrics { + ruleExecutionsTotal: Counter; ruleExecutions: Counter; + ruleFailures: Counter; + ruleDuration: Histogram; constructor(meter: Meter) { + this.ruleExecutionsTotal = meter.createCounter('ruleExecutionsTotal'); this.ruleExecutions = meter.createCounter('ruleExecutions'); + this.ruleFailures = meter.createCounter('ruleFailures'); + this.ruleDuration = meter.createHistogram('ruleDuration'); } } diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index fc09e7c80802b..f701c8997ac93 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -785,6 +785,7 @@ export class TaskRunner< if (null != duration) { executionStatus.lastDuration = nanosToMillis(duration); monitoringHistory.duration = executionStatus.lastDuration; + this.metrics.ruleDuration.record(executionStatus.lastDuration); } // if executionStatus indicates an error, fill in fields in @@ -802,9 +803,11 @@ export class TaskRunner< if (!this.cancelled) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_EXECUTIONS); // NOTE: Using the typesafe opentelemetry counter here directly - this.metrics.ruleExecutions.add(1); + this.metrics.ruleExecutionsTotal.add(1); + this.metrics.ruleExecutions.add(1, { ruleType: this.ruleType.id }); if (executionStatus.error) { this.inMemoryMetrics.increment(IN_MEMORY_METRICS.RULE_FAILURES); + this.metrics.ruleFailures.add(1); } this.logger.debug( `Updating rule task for ${this.ruleType.id} rule with id ${ruleId} - ${JSON.stringify( From d02dcebe2d16c5b3e6b7a6118a7170269b172f46 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Tue, 14 Jun 2022 14:08:17 +0900 Subject: [PATCH 30/31] Test against real metrics with test meter --- .../alerting/server/monitoring/metrics.mock.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts b/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts index 280acabac2329..129298e63e90f 100644 --- a/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts +++ b/x-pack/plugins/alerting/server/monitoring/metrics.mock.ts @@ -5,14 +5,9 @@ * 2.0. */ -function createInMemoryMetricsMock() { - return jest.fn().mockImplementation(() => { - return { - ruleExecutions: { add: jest.fn() }, - }; - }); -} +import { metrics } from '@opentelemetry/api-metrics'; +import { Metrics } from './metrics'; export const metricsMock = { - create: createInMemoryMetricsMock(), + create: () => new Metrics(metrics.getMeter('kibana.alerting.metrics.mock')), }; From 371296aaef8e062d5d170bde1cd512a35abb11e0 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Tue, 14 Jun 2022 15:16:17 +0900 Subject: [PATCH 31/31] Add OTLP header config --- .../plugins/monitoring_collection/server/config.ts | 1 + .../plugins/monitoring_collection/server/plugin.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring_collection/server/config.ts b/x-pack/plugins/monitoring_collection/server/config.ts index f6b5a62925d7e..6efe92986ddd5 100644 --- a/x-pack/plugins/monitoring_collection/server/config.ts +++ b/x-pack/plugins/monitoring_collection/server/config.ts @@ -13,6 +13,7 @@ export const configSchema = schema.object({ metrics: schema.object({ otlp: schema.object({ url: schema.maybe(schema.string()), + headers: schema.maybe(schema.recordOf(schema.string(), schema.string())), exportIntervalMillis: schema.number({ defaultValue: 10000 }), }), prometheus: schema.object({ diff --git a/x-pack/plugins/monitoring_collection/server/plugin.ts b/x-pack/plugins/monitoring_collection/server/plugin.ts index 97c7136e2663a..7f42f99c5d95f 100644 --- a/x-pack/plugins/monitoring_collection/server/plugin.ts +++ b/x-pack/plugins/monitoring_collection/server/plugin.ts @@ -126,6 +126,7 @@ export class MonitoringCollectionPlugin implements Plugin